Function prepare_wmt_data

chatbotv4/data_utils.py:211–239

Get WMT data into data_dir, create vocabularies and tokenize data. Args: data_dir: directory in which the data sets will be stored. en_vocabulary_size: size of the English vocabulary to create and use. fr_vocabulary_size: size of the French vocabulary to create and use. tokenizer:

(data_dir, en_vocabulary_size, fr_vocabulary_size, tokenizer=None)

def prepare_wmt_data(data_dir, en_vocabulary_size, fr_vocabulary_size, tokenizer=None):
  """Get WMT data into data_dir, create vocabularies and tokenize data.

  Args:
    data_dir: directory in which the data sets will be stored.
    en_vocabulary_size: size of the English vocabulary to create and use.
    fr_vocabulary_size: size of the French vocabulary to create and use.
    tokenizer: a function to use to tokenize each data sentence;
      if None, basic_tokenizer will be used.

  Returns:
    A tuple of 6 elements:
      (1) path to the token-ids for English training data-set,
      (2) path to the token-ids for French training data-set,
      (3) path to the token-ids for English development data-set,
      (4) path to the token-ids for French development data-set,
      (5) path to the English vocabulary file,
      (6) path to the French vocabulary file.
  """
  # Get wmt data to the specified directory.
  train_path = get_wmt_enfr_train_set(data_dir)
  dev_path = get_wmt_enfr_dev_set(data_dir)

  input_train_path = train_path + ".input"
  output_train_path = train_path + ".output"
  input_dev_path = dev_path + ".input"
  output_dev_path = dev_path + ".output"
  return prepare_data(data_dir, input_train_path, output_train_path, input_dev_path, output_dev_path, en_vocabulary_size,
                      fr_vocabulary_size, tokenizer)


def prepare_data(data_dir, input_train_path, output_train_path, input_dev_path, output_dev_path, input_vocabulary_size,