Function build_partitioned_load

testdata/bin/generate-schema-statements.py:551–580

(insert, re_match, can_hint, params)

def build_partitioned_load(insert, re_match, can_hint, params):
  part_col = re_match.group(1)
  min_val = int(re_match.group(2))
  max_val = int(re_match.group(3))
  batch = int(re_match.group(4))
  insert = PARTITIONED_INSERT_RE.sub("", insert)
  statements = []
  params[VAR_HINT] = HINT_SHUFFLE if can_hint else ''
  first_part = min_val
  while first_part < max_val:
    if first_part + batch >= max_val:
      # This is the last batch
      if first_part == min_val:
        # There is only 1 batch in this insert. No predicate needed.
        params[VAR_PART_PREDICATE] = ''
      else:
        # Insert the remaining partitions + NULL partition
        params[VAR_PART_PREDICATE] = "WHERE {0} <= {1} OR {2} IS NULL".format(
          first_part, part_col, part_col)
    elif first_part == min_val:
      # This is the first batch.
      params[VAR_PART_PREDICATE] = "WHERE {0} < {1}".format(
        part_col, first_part + batch)
    else:
      # This is the middle batch.
      params[VAR_PART_PREDICATE] = "WHERE {0} <= {1} AND {2} < {3}".format(
        first_part, part_col, part_col, first_part + batch)
    statements.append(insert.format(**params))
    first_part += batch
  return "\n".join(statements)


def build_insert_into_statement(insert, db_name, db_suffix, table_name, file_format,