Function resplit_data

language/orqa/preprocessing/create_data_splits.py:41–50

Resplit a dataset of jsonlines with QA pairs.

(input_path, output_path, keep_fn)

def resplit_data(input_path, output_path, keep_fn):
  """Resplit a dataset of jsonlines with QA pairs."""
  count = 0
  with tf.io.gfile.GFile(output_path, "w") as output_file:
    for line in tf.io.gfile.GFile(input_path):
      example = json.loads(line)
      if keep_fn(example["question"]):
        output_file.write(line)
        count += 1
  tf.logging.info("Wrote {} examples to {}".format(count, output_path))


def resplit_nq():