()
| 143 | |
| 144 | |
| 145 | def test_tpch(): |
| 146 | print_msg(f"Generating TPCH (sf={scale_factor})") |
| 147 | tpch = TPCHData(scale_factor) |
| 148 | |
| 149 | ## -------- Benchmark converting LineItem to different formats --------- |
| 150 | |
| 151 | def fetch_native(rel: duckdb.DuckDBPyRelation): |
| 152 | return rel.fetchall() |
| 153 | |
| 154 | def fetch_pandas(rel: duckdb.DuckDBPyRelation): |
| 155 | return rel.df() |
| 156 | |
| 157 | def fetch_arrow(rel: duckdb.DuckDBPyRelation): |
| 158 | return rel.arrow() |
| 159 | |
| 160 | COLLECTORS = {'native': fetch_native, 'pandas': fetch_pandas, 'arrow': fetch_arrow} |
| 161 | # For every collector, load lineitem 'nrun' times |
| 162 | for collector in COLLECTORS: |
| 163 | result: BenchmarkResult = tpch.load_lineitem(COLLECTORS[collector], collector + "_load_lineitem") |
| 164 | print_msg(result.name) |
| 165 | print_msg(collector) |
| 166 | result.write() |
| 167 | |
| 168 | ## ------- Benchmark running TPCH queries on top of different formats -------- |
| 169 | |
| 170 | def convert_pandas(conn: duckdb.DuckDBPyConnection, table_name: str): |
| 171 | return conn.execute(f"SELECT * FROM {table_name}").df() |
| 172 | |
| 173 | def convert_arrow(conn: duckdb.DuckDBPyConnection, table_name: str): |
| 174 | df = convert_pandas(conn, table_name) |
| 175 | return pa.Table.from_pandas(df) |
| 176 | |
| 177 | CONVERTORS = {'pandas': convert_pandas, 'arrow': convert_arrow} |
| 178 | # Convert TPCH data to the right format, then run TPCH queries on that data |
| 179 | for convertor in CONVERTORS: |
| 180 | tables = tpch.get_tables(CONVERTORS[convertor]) |
| 181 | tester = TPCHBenchmarker(convertor) |
| 182 | tester.register_tables(tables) |
| 183 | collector = COLLECTORS[convertor] |
| 184 | result: BenchmarkResult = tester.run_tpch(collector, f"{convertor}tpch") |
| 185 | result.write() |
| 186 | |
| 187 | |
| 188 | def generate_string(seed: int): |
no test coverage detected