| 26 | const string csv_extensions[5] = {csv, tsv, csv_gz, csv_zst, tbl_zst}; |
| 27 | |
| 28 | bool RunVariableBuffer(const string &path, idx_t buffer_size, bool set_temp_dir, |
| 29 | ColumnDataCollection *ground_truth = nullptr, const string &add_parameters = "") { |
| 30 | DuckDB db(nullptr); |
| 31 | Connection multi_conn(db); |
| 32 | if (set_temp_dir) { |
| 33 | multi_conn.Query("PRAGMA temp_directory='offload.tmp'"); |
| 34 | } |
| 35 | multi_conn.Query("SET preserve_insertion_order=false;"); |
| 36 | duckdb::unique_ptr<MaterializedQueryResult> variable_buffer_size_result = |
| 37 | multi_conn.Query("SELECT * FROM read_csv_auto('" + path + "'" + add_parameters + |
| 38 | ", buffer_size = " + to_string(buffer_size) + ") ORDER BY ALL"); |
| 39 | bool variable_buffer_size_passed; |
| 40 | ColumnDataCollection *result = nullptr; |
| 41 | if (variable_buffer_size_result->HasError()) { |
| 42 | variable_buffer_size_passed = false; |
| 43 | } else { |
| 44 | variable_buffer_size_passed = true; |
| 45 | result = &variable_buffer_size_result->Collection(); |
| 46 | } |
| 47 | if (!ground_truth && !variable_buffer_size_passed) { |
| 48 | // Two wrongs can make a right |
| 49 | return true; |
| 50 | } |
| 51 | if (!ground_truth) { |
| 52 | //! oh oh, this should not pass |
| 53 | std::cout << path << " Failed on max buffer but succeeded on variable buffer reading" << '\n'; |
| 54 | return false; |
| 55 | } |
| 56 | if (!variable_buffer_size_passed) { |
| 57 | std::cout << path << " Variable Buffer failed" << '\n'; |
| 58 | std::cout << path << " Buffer Size: " << to_string(buffer_size) << '\n'; |
| 59 | std::cout << variable_buffer_size_result->GetError() << '\n'; |
| 60 | return false; |
| 61 | } |
| 62 | // Results do not match |
| 63 | string error_message; |
| 64 | if (!ColumnDataCollection::ResultEquals(*ground_truth, *result, error_message, false)) { |
| 65 | std::cout << "truth: " << ground_truth->Count() << std::endl; |
| 66 | std::cout << "resul: " << result->Count() << std::endl; |
| 67 | |
| 68 | std::cout << path << " Buffer Size: " << to_string(buffer_size) << '\n'; |
| 69 | std::cout << error_message << '\n'; |
| 70 | return false; |
| 71 | } |
| 72 | return true; |
| 73 | } |
| 74 | |
| 75 | bool RunFull(const std::string &path, std::set<std::string> *skip = nullptr, const string &add_parameters = "", |
| 76 | bool set_temp_dir = false) { |