(table_template, table_name, db_name, db_suffix,
file_format, hdfs_location, force_reload, is_hive_stmt,
scale_factor="")
| 275 | |
| 276 | |
| 277 | def build_create_statement(table_template, table_name, db_name, db_suffix, |
| 278 | file_format, hdfs_location, force_reload, is_hive_stmt, |
| 279 | scale_factor=""): |
| 280 | create_stmt = '' |
| 281 | if (force_reload): |
| 282 | tbl_type = 'TABLE' |
| 283 | if table_template.upper().strip().startswith('CREATE VIEW'): |
| 284 | tbl_type = 'VIEW' |
| 285 | create_stmt += 'DROP %s IF EXISTS %s%s.%s;\n' \ |
| 286 | % (tbl_type, db_name, db_suffix, table_name) |
| 287 | # hbase / kudu tables are external, and not read from hdfs. We don't need an |
| 288 | # hdfs_location. |
| 289 | if file_format in ['hbase', 'kudu']: |
| 290 | hdfs_location = str() |
| 291 | # Remove location part from the format string |
| 292 | table_template = table_template.replace("LOCATION '{hdfs_location}'", "") |
| 293 | |
| 294 | params = build_replacement_params( |
| 295 | db_name, |
| 296 | db_suffix, |
| 297 | table_name, |
| 298 | file_format=FILE_FORMAT_TO_STORED_AS_MAP[file_format], |
| 299 | hdfs_location=hdfs_location, |
| 300 | scale_factor=scale_factor) |
| 301 | stmt = table_template.format(**params) |
| 302 | # Apache Hive 3.1 doesn't support "STORED BY ICEBERG STORED AS AVRO" and |
| 303 | # "STORED AS JSONFILE" (HIVE-25162, HIVE-19899) |
| 304 | if is_hive_stmt and os.environ['USE_APACHE_HIVE_3'] == "true": |
| 305 | if "STORED AS JSONFILE" in stmt: |
| 306 | stmt = stmt.replace("STORED AS JSONFILE", |
| 307 | "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.JsonSerDe'") |
| 308 | elif "STORED BY ICEBERG" in stmt: |
| 309 | if "STORED AS" not in stmt: |
| 310 | stmt = stmt.replace( |
| 311 | "STORED BY ICEBERG", |
| 312 | "STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'") |
| 313 | else: |
| 314 | assert "TBLPROPERTIES" not in stmt,\ |
| 315 | ("Cannot convert STORED BY ICEBERG STORED AS file_format with TBLPROPERTIES " |
| 316 | "also in the statement:\n" + stmt) |
| 317 | iceberg_file_format = re.search(r"STORED AS (\w+)", stmt).group(1) |
| 318 | # TBLPROPERTIES should be put after LOCATION |
| 319 | if "LOCATION" not in stmt: |
| 320 | stmt = re.sub( |
| 321 | r"STORED BY ICEBERG\s+STORED AS \w+", |
| 322 | ("STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'" |
| 323 | " TBLPROPERTIES('write.format.default'='{}')").format(iceberg_file_format), |
| 324 | stmt) |
| 325 | else: |
| 326 | stmt = re.sub( |
| 327 | r"STORED BY ICEBERG\s+STORED AS \w+", |
| 328 | "STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'", |
| 329 | stmt) |
| 330 | loc_clause = re.search(r"LOCATION ['\"][^\s]+['\"]", stmt).group(0) |
| 331 | stmt = stmt.replace(loc_clause, |
| 332 | loc_clause + " TBLPROPERTIES('write.format.default'='{}')" |
| 333 | .format(iceberg_file_format)) |
| 334 | create_stmt += stmt |
no test coverage detected