Test LOAD DATA INPATH for Iceberg tables, the first part of this method inits the target directory, copies existing test data to HDFS. The second part runs the test cases then cleans up the test directory.
(self, vector, unique_database)
| 1370 | unique_database) |
| 1371 | |
| 1372 | def test_load(self, vector, unique_database): |
| 1373 | """Test LOAD DATA INPATH for Iceberg tables, the first part of this method inits the |
| 1374 | target directory, copies existing test data to HDFS. The second part runs the test |
| 1375 | cases then cleans up the test directory. |
| 1376 | """ |
| 1377 | # Test 1-6 init: target orc/parquet file and directory |
| 1378 | SRC_DIR = os.path.join(os.environ['IMPALA_HOME'], |
| 1379 | "testdata/data/iceberg_test/iceberg_mixed_file_format_test/data/{0}") |
| 1380 | DST_DIR = "/tmp/" + unique_database + "/parquet/" |
| 1381 | self.filesystem_client.make_dir(DST_DIR, permission=777) |
| 1382 | file_parq1 = "00000-0-data-gfurnstahl_20220906113044_157fc172-f5d3-4c70-8653-" \ |
| 1383 | "fff150b6136a-job_16619542960420_0002-1-00001.parquet" |
| 1384 | file_parq2 = "00000-0-data-gfurnstahl_20220906114830_907f72c7-36ac-4135-8315-" \ |
| 1385 | "27ff880faff0-job_16619542960420_0004-1-00001.parquet" |
| 1386 | self.filesystem_client.copy_from_local(SRC_DIR.format(file_parq1), DST_DIR) |
| 1387 | self.filesystem_client.copy_from_local(SRC_DIR.format(file_parq2), DST_DIR) |
| 1388 | DST_DIR = "/tmp/" + unique_database + "/orc/" |
| 1389 | self.filesystem_client.make_dir(DST_DIR, permission=777) |
| 1390 | file_orc1 = "00000-0-data-gfurnstahl_20220906113255_8d49367d-e338-4996-ade5-" \ |
| 1391 | "ee500a19c1d1-job_16619542960420_0003-1-00001.orc" |
| 1392 | file_orc2 = "00000-0-data-gfurnstahl_20220906114900_9c1b7b46-5643-428f-a007-" \ |
| 1393 | "519c5500ed04-job_16619542960420_0004-1-00001.orc" |
| 1394 | self.filesystem_client.copy_from_local(SRC_DIR.format(file_orc1), DST_DIR) |
| 1395 | self.filesystem_client.copy_from_local(SRC_DIR.format(file_orc2), DST_DIR) |
| 1396 | # Test 7 init: overwrite |
| 1397 | DST_DIR = "/tmp/" + unique_database + "/overwrite/" |
| 1398 | self.filesystem_client.make_dir(DST_DIR, permission=777) |
| 1399 | self.filesystem_client.copy_from_local(SRC_DIR.format(file_parq1), DST_DIR) |
| 1400 | # Test 8 init: mismatching parquet schema format |
| 1401 | SRC_DIR = os.path.join(os.environ['IMPALA_HOME'], "testdata/data/iceberg_test/" |
| 1402 | "iceberg_partitioned/data/event_time_hour=2020-01-01-08/action=view/{0}") |
| 1403 | DST_DIR = "/tmp/" + unique_database + "/mismatching_schema/" |
| 1404 | self.filesystem_client.make_dir(DST_DIR, permission=777) |
| 1405 | file = "00001-1-b975a171-0911-47c2-90c8-300f23c28772-00000.parquet" |
| 1406 | self.filesystem_client.copy_from_local(SRC_DIR.format(file), DST_DIR) |
| 1407 | # Test 9 init: partitioned |
| 1408 | DST_DIR = "/tmp/" + unique_database + "/partitioned/" |
| 1409 | self.filesystem_client.make_dir(DST_DIR, permission=777) |
| 1410 | self.filesystem_client.copy_from_local(SRC_DIR.format(file), DST_DIR) |
| 1411 | # Test 10 init: hidden files |
| 1412 | DST_DIR = "/tmp/" + unique_database + "/hidden/" |
| 1413 | self.filesystem_client.make_dir(DST_DIR, permission=777) |
| 1414 | self.filesystem_client.create_file(DST_DIR + "_hidden.1", "Test data 123") |
| 1415 | self.filesystem_client.create_file(DST_DIR + "_hidden_2.1", "Test data 123") |
| 1416 | self.filesystem_client.create_file(DST_DIR + ".hidden_3", "Test data 123") |
| 1417 | self.filesystem_client.create_file(DST_DIR + ".hidden_4.1", "Test data 123") |
| 1418 | self.filesystem_client.copy_from_local(SRC_DIR.format(file), DST_DIR) |
| 1419 | |
| 1420 | # Init test table |
| 1421 | create_iceberg_table_from_directory(self.client, unique_database, |
| 1422 | "iceberg_mixed_file_format_test", "parquet") |
| 1423 | |
| 1424 | # Execute tests |
| 1425 | self.run_test_case('QueryTest/iceberg-load', vector, use_db=unique_database) |
| 1426 | # Clean up temporary directory |
| 1427 | self.filesystem_client.delete_file_dir("/tmp/{0}".format(unique_database), True) |
| 1428 | |
| 1429 | def test_table_sampling(self, vector): |
nothing calls this directly
no test coverage detected