| 121 | // Parsing raw csv data into FieldLocation descriptors. |
| 122 | template<bool DELIMITED_TUPLES> |
| 123 | Status DelimitedTextParser<DELIMITED_TUPLES>::ParseFieldLocations(int max_tuples, |
| 124 | int64_t remaining_len, char** byte_buffer_ptr, char** row_end_locations, |
| 125 | FieldLocation* field_locations, |
| 126 | int* num_tuples, int* num_fields, char** next_column_start) { |
| 127 | // Start of this batch. |
| 128 | *next_column_start = *byte_buffer_ptr; |
| 129 | // If there was a '\r' at the end of the last batch, set the offset to |
| 130 | // just before the beginning. Otherwise make it invalid. |
| 131 | if (last_row_delim_offset_ == 0) { |
| 132 | last_row_delim_offset_ = remaining_len; |
| 133 | } else { |
| 134 | last_row_delim_offset_ = -1; |
| 135 | } |
| 136 | |
| 137 | #ifdef __x86_64__ |
| 138 | if (CpuInfo::IsSupported(CpuInfo::SSE4_2)) { |
| 139 | if (process_escapes_) { |
| 140 | RETURN_IF_ERROR(ParseSse<true>(max_tuples, &remaining_len, byte_buffer_ptr, |
| 141 | row_end_locations, field_locations, num_tuples, num_fields, next_column_start)); |
| 142 | } else { |
| 143 | RETURN_IF_ERROR(ParseSse<false>(max_tuples, &remaining_len, byte_buffer_ptr, |
| 144 | row_end_locations, field_locations, num_tuples, num_fields, next_column_start)); |
| 145 | } |
| 146 | } |
| 147 | #endif |
| 148 | |
| 149 | if (*num_tuples == max_tuples) return Status::OK(); |
| 150 | |
| 151 | // Handle the remaining characters |
| 152 | while (remaining_len > 0) { |
| 153 | bool new_tuple = false; |
| 154 | bool new_col = false; |
| 155 | if (DELIMITED_TUPLES) unfinished_tuple_ = true; |
| 156 | |
| 157 | if (!last_char_is_escape_) { |
| 158 | if (DELIMITED_TUPLES && (**byte_buffer_ptr == tuple_delim_ || |
| 159 | (tuple_delim_ == '\n' && **byte_buffer_ptr == '\r'))) { |
| 160 | new_tuple = true; |
| 161 | new_col = true; |
| 162 | } else if (**byte_buffer_ptr == field_delim_ |
| 163 | || **byte_buffer_ptr == collection_item_delim_) { |
| 164 | new_col = true; |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | if (process_escapes_ && **byte_buffer_ptr == escape_char_) { |
| 169 | current_column_has_escape_ = true; |
| 170 | last_char_is_escape_ = !last_char_is_escape_; |
| 171 | } else { |
| 172 | last_char_is_escape_ = false; |
| 173 | } |
| 174 | |
| 175 | if (new_tuple) { |
| 176 | if (last_row_delim_offset_ == remaining_len && **byte_buffer_ptr == '\n') { |
| 177 | // If the row ended in \r\n then move to the \n |
| 178 | ++*next_column_start; |
| 179 | } else { |
| 180 | RETURN_IF_ERROR(AddColumn<true>(*byte_buffer_ptr - *next_column_start, |