| 63 | } |
| 64 | |
| 65 | TEST(DelimitedTextParser, Basic) { |
| 66 | const char TUPLE_DELIM = '|'; |
| 67 | const char FIELD_DELIM = ','; |
| 68 | const char COLLECTION_DELIM = ','; |
| 69 | const char ESCAPE_CHAR = '@'; |
| 70 | |
| 71 | const int NUM_COLS = 1; |
| 72 | |
| 73 | // Test without escape |
| 74 | bool is_materialized_col[NUM_COLS]; |
| 75 | for (int i = 0; i < NUM_COLS; ++i) is_materialized_col[i] = true; |
| 76 | |
| 77 | TupleDelimitedTextParser no_escape_parser(NUM_COLS, 0, is_materialized_col, |
| 78 | TUPLE_DELIM, FIELD_DELIM, COLLECTION_DELIM); |
| 79 | // Note that only complete tuples "count" |
| 80 | Validate(&no_escape_parser, "no_delims", -1, TUPLE_DELIM, 0, 0); |
| 81 | Validate(&no_escape_parser, "abc||abc", 4, TUPLE_DELIM, 1, 1); |
| 82 | Validate(&no_escape_parser, "|abcd", 1, TUPLE_DELIM, 0, 0); |
| 83 | Validate(&no_escape_parser, "a|bcd", 2, TUPLE_DELIM, 0, 0); |
| 84 | |
| 85 | // Test with escape char |
| 86 | TupleDelimitedTextParser escape_parser(NUM_COLS, 0, is_materialized_col, |
| 87 | TUPLE_DELIM, FIELD_DELIM, COLLECTION_DELIM, |
| 88 | ESCAPE_CHAR); |
| 89 | Validate(&escape_parser, "a@|a|bcd", 5, TUPLE_DELIM, 0, 0); |
| 90 | Validate(&escape_parser, "a@@|a|bcd", 4, TUPLE_DELIM, 1, 1); |
| 91 | Validate(&escape_parser, "a@@@|a|bcd", 7, TUPLE_DELIM, 0, 0); |
| 92 | Validate(&escape_parser, "a@@@@|a|bcd", 6, TUPLE_DELIM, 1, 1); |
| 93 | Validate(&escape_parser, "a|@@@|a|bcd", 2, TUPLE_DELIM, 1, 1); |
| 94 | |
| 95 | // // The parser doesn't support this case. |
| 96 | // // TODO: update test when it is fixed |
| 97 | // Validate(&escape_parser, "@|no_delims", -1, TUPLE_DELIM); |
| 98 | |
| 99 | // Test null characters |
| 100 | const string str1("\0no_delims", 10); |
| 101 | const string str2("ab\0||abc", 8); |
| 102 | const string str3("\0|\0|\0", 5); |
| 103 | const string str4("abc|\0a|abc", 10); |
| 104 | const string str5("\0|aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 32); |
| 105 | Validate(&no_escape_parser, str1, -1, TUPLE_DELIM, 0, 0); |
| 106 | Validate(&no_escape_parser, str2, 4, TUPLE_DELIM, 1, 1); |
| 107 | Validate(&no_escape_parser, str3, 2, TUPLE_DELIM, 1, 1); |
| 108 | Validate(&no_escape_parser, str4, 4, TUPLE_DELIM, 1, 1); |
| 109 | Validate(&no_escape_parser, str5, 2, TUPLE_DELIM, 0, 0); |
| 110 | |
| 111 | const string str6("\0@|\0|\0", 6); |
| 112 | const string str7("\0@@|\0|\0", 6); |
| 113 | const string str8("\0@\0@|\0|\0", 8); |
| 114 | const string str9("\0@||aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 34); |
| 115 | Validate(&escape_parser, str6, 5, TUPLE_DELIM, 0, 0); |
| 116 | Validate(&escape_parser, str7, 4, TUPLE_DELIM, 1, 1); |
| 117 | Validate(&escape_parser, str8, 7, TUPLE_DELIM, 0, 0); |
| 118 | Validate(&escape_parser, str9, 4, TUPLE_DELIM, 0, 0); |
| 119 | } |
| 120 | |
| 121 | TEST(DelimitedTextParser, Fields) { |
| 122 | const char TUPLE_DELIM = '|'; |