Runs query files and captures results from the specified workload(s) The usage is: 1) Initialize WorkloadRunner with desired execution parameters. 2) Call workload_runner.run() Internally, for each workload, this module looks up and parses that workload's query files and reads the wo
| 36 | |
| 37 | |
| 38 | class WorkloadRunner(object): |
| 39 | """Runs query files and captures results from the specified workload(s) |
| 40 | |
| 41 | The usage is: |
| 42 | 1) Initialize WorkloadRunner with desired execution parameters. |
| 43 | 2) Call workload_runner.run() |
| 44 | |
| 45 | Internally, for each workload, this module looks up and parses that workload's |
| 46 | query files and reads the workload's test vector to determine what combination(s) |
| 47 | of file format / compression to run with. |
| 48 | |
| 49 | Args: |
| 50 | workload (Workload) |
| 51 | scale_factor (str): eg. "300gb" |
| 52 | config (WorkloadConfig) |
| 53 | |
| 54 | Attributes: |
| 55 | workload (Workload) |
| 56 | scale_factor (str): eg. "300gb" |
| 57 | config (WorkloadConfig) |
| 58 | exit_on_error (boolean) |
| 59 | results (list of ImpalaQueryResult) |
| 60 | _test_vectors (list of ?) |
| 61 | """ |
| 62 | |
| 63 | def __init__(self, workload, scale_factor, config): |
| 64 | self.workload = workload |
| 65 | self.scale_factor = scale_factor |
| 66 | self.config = config |
| 67 | self.exit_on_error = not self.config.continue_on_query_error |
| 68 | if self.config.verbose: LOG.setLevel(level=logging.DEBUG) |
| 69 | self._generate_test_vectors() |
| 70 | self._results = list() |
| 71 | |
| 72 | @property |
| 73 | def results(self): |
| 74 | return self._results |
| 75 | |
| 76 | def _generate_test_vectors(self): |
| 77 | """Generate test vector objects |
| 78 | |
| 79 | If the user has specified a set for table_formats, generate them, otherwise generate |
| 80 | vectors for all table formats within the specified exploration strategy. |
| 81 | """ |
| 82 | self._test_vectors = [] |
| 83 | if self.config.table_formats: |
| 84 | dataset = get_dataset_from_workload(self.workload.name) |
| 85 | for tf in self.config.table_formats: |
| 86 | self._test_vectors.append(TableFormatInfo.create_from_string(dataset, tf)) |
| 87 | else: |
| 88 | vectors = load_table_info_dimension(self.workload.name, |
| 89 | self.config.exploration_strategy) |
| 90 | self._test_vectors = [vector.value for vector in vectors] |
| 91 | |
| 92 | def _create_executor(self, executor_name): |
| 93 | query_options = { |
| 94 | 'impala_jdbc': lambda: ( |
| 95 | execute_using_jdbc, |