()
| 303 | |
| 304 | |
| 305 | def main(): |
| 306 | logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%H:%M:%S') |
| 307 | LOG.setLevel(logging.DEBUG) |
| 308 | |
| 309 | # Having the actual command line at the top of each data-load-* log can help |
| 310 | # when debugging dataload issues. |
| 311 | # |
| 312 | LOG.debug(' '.join(sys.argv)) |
| 313 | |
| 314 | all_workloads = available_workloads(WORKLOAD_DIR) |
| 315 | workloads = [] |
| 316 | if options.workloads is None: |
| 317 | LOG.error("At least one workload name must be specified.") |
| 318 | parser.print_help() |
| 319 | sys.exit(1) |
| 320 | elif options.workloads == 'all': |
| 321 | LOG.info('Loading data for all workloads.') |
| 322 | workloads = all_workloads |
| 323 | else: |
| 324 | workloads = options.workloads.split(",") |
| 325 | validate_workloads(all_workloads, workloads) |
| 326 | |
| 327 | LOG.info('Starting data load for the following workloads: ' + ', '.join(workloads)) |
| 328 | LOG.info('Running with {0} threads'.format(options.num_processes)) |
| 329 | |
| 330 | # Note: The processes are in whatever the caller's directory is, so all paths |
| 331 | # passed to the pool need to be absolute paths. This will allow the pool |
| 332 | # to be used for different workloads (and thus different directories) |
| 333 | # simultaneously. |
| 334 | thread_pool = ThreadPool(processes=options.num_processes) |
| 335 | loading_time_map = collections.defaultdict(float) |
| 336 | for workload in workloads: |
| 337 | start_time = time.time() |
| 338 | dataset = get_dataset_for_workload(workload) |
| 339 | run_dataset_preload(dataset) |
| 340 | # This script is tightly coupled with testdata/bin/generate-schema-statements.py |
| 341 | # Specifically, this script is expecting the following: |
| 342 | # 1. generate-schema-statements.py generates files and puts them in the |
| 343 | # directory ${IMPALA_DATA_LOADING_SQL_DIR}/${workload} |
| 344 | # (e.g. ${IMPALA_HOME}/logs/data_loading/sql/tpch) |
| 345 | # 2. generate-schema-statements.py populates the subdirectory |
| 346 | # avro_schemas/${workload} with JSON files specifying the Avro schema for the |
| 347 | # tables being loaded. |
| 348 | # 3. generate-schema-statements.py uses a particular naming scheme to distinguish |
| 349 | # between SQL files of different load phases. |
| 350 | # |
| 351 | # Using the following variables: |
| 352 | # workload_exploration = ${workload}-${exploration_strategy} and |
| 353 | # file_format_suffix = ${file_format}-${codec}-${compression_type} |
| 354 | # |
| 355 | # A. Impala table creation scripts run in Impala to create tables, partitions, |
| 356 | # and views. There is one for each file format. They take the form: |
| 357 | # create-${workload_exploration}-impala-generated-${file_format_suffix}.sql |
| 358 | # |
| 359 | # B. Hive creation/load scripts run in Hive to load data into tables and create |
| 360 | # tables or views that Impala does not support. There is one for each |
| 361 | # file format. They take the form: |
| 362 | # load-${workload_exploration}-hive-generated-${file_format_suffix}.sql |
no test coverage detected