Returns a list of queries for the given workload. 'workload' should either be 'tpch', 'tpcds', 'tpch_nested', or 'targeted-perf'. The types of queries that are returned: - 'standard' queries, i.e. from the spec for that workload. These queries will have filenames like '{workload}-q*.test'
(workload, include_stress_queries=False, query_name_filters=[])
| 371 | |
| 372 | |
| 373 | def load_tpc_queries(workload, include_stress_queries=False, query_name_filters=[]): |
| 374 | """ |
| 375 | Returns a list of queries for the given workload. 'workload' should either be 'tpch', |
| 376 | 'tpcds', 'tpch_nested', or 'targeted-perf'. The types of queries that are returned: |
| 377 | - 'standard' queries, i.e. from the spec for that workload. These queries will have |
| 378 | filenames like '{workload}-q*.test' and one query per file. |
| 379 | - 'stress' queries, if 'include_stress_queries' is true, which run against data from |
| 380 | the workkload but were designed to stress specific aspects of Impala. These queries |
| 381 | have filenames like '{workload}-stress-*.test' and may have multiple queries per file. |
| 382 | - 'targeted' queries, if the workload is 'targeted-perf', which have no restrictions. |
| 383 | The returned queries are filtered according to 'query_name_filters', a list of query |
| 384 | name regexes, if specified. |
| 385 | All queries are required to have a name specified, i.e. each test case should have: |
| 386 | ---- QUERY: WORKLOAD-<QUERY_NAME> |
| 387 | """ |
| 388 | LOG.info("Loading %s queries", workload) |
| 389 | queries = dict() |
| 390 | query_dir = os.path.join( |
| 391 | os.environ['IMPALA_HOME'], "testdata", "workloads", workload, "queries") |
| 392 | # Check whether the workload name corresponds to an existing directory. |
| 393 | if not os.path.isdir(query_dir): |
| 394 | raise ValueError("Workload %s not found in %s" % (workload, query_dir)) |
| 395 | |
| 396 | # IMPALA-6715 and others from the past: This pattern enforces the queries we actually |
| 397 | # find. Both workload directories contain other queries that are not part of the TPC |
| 398 | # spec. |
| 399 | file_workload = workload |
| 400 | if workload in ["tpcds", "tpcds_partitioned"]: |
| 401 | # TPCDS is assumed to always use decimal_v2, which is the default since 3.0 |
| 402 | file_workload = "tpcds-decimal_v2" |
| 403 | if include_stress_queries: |
| 404 | file_name_pattern = re.compile(r"^{0}-(q.*|stress-.*).test$".format(file_workload)) |
| 405 | else: |
| 406 | file_name_pattern = re.compile(r"^{0}-(q.*).test$".format(file_workload)) |
| 407 | |
| 408 | query_name_prefix = workload.upper() |
| 409 | if workload == "tpcds_partitioned": |
| 410 | # 'tpcds_partitioned' symlink to the same queries set of 'tpcds'. |
| 411 | query_name_prefix = "TPCDS" |
| 412 | query_name_pattern = re.compile(r"^{0}-(.*)$".format(query_name_prefix)) |
| 413 | if workload == "tpch_nested": |
| 414 | query_name_pattern = re.compile(r"^TPCH-(.*)$") |
| 415 | |
| 416 | if workload == "targeted-perf": |
| 417 | # We don't enforce any restrictions on targeted-perf queries. |
| 418 | file_name_pattern = re.compile(r"(.*)") |
| 419 | query_name_pattern = re.compile(r"(.*)") |
| 420 | |
| 421 | if query_name_filters: |
| 422 | query_name_filters = list(map(str.strip, query_name_filters)) |
| 423 | else: |
| 424 | query_name_filters = [] |
| 425 | filter_regex = re.compile(r'|'.join(['^%s$' % n for n in query_name_filters]), re.I) |
| 426 | |
| 427 | for query_file in os.listdir(query_dir): |
| 428 | is_standard = "stress" not in query_file and workload != "targeted-perf" |
| 429 | file_match = file_name_pattern.search(query_file) |
| 430 | if not file_match: |
no test coverage detected