Function get_data

nlp_class3/memory_network.py:159–214

(challenge_type)

# we want to load both the single supporting fact data
# and the two supporting fact data later
def get_data(challenge_type):
  # input should either be 'single_supporting_fact_10k' or 'two_supporting_facts_10k'
  challenge = challenges[challenge_type]


  # returns a list of triples of:
  # (story, question, answer)
  # story is a list of sentences
  # question is a sentence
  # answer is a word
  train_stories = get_stories(tar.extractfile(challenge.format('train')))
  test_stories = get_stories(tar.extractfile(challenge.format('test')))


  # group all the stories together
  stories = train_stories + test_stories

  # so we can get the max length of each story, of each sentence, and of each question
  story_maxlen = max((len(s) for x, _, _ in stories for s in x))
  story_maxsents = max((len(x) for x, _, _ in stories))
  query_maxlen = max(len(x) for _, x, _ in stories)

  # Create vocabulary of corpus and find size, including a padding element.
  vocab = sorted(set(flatten(stories)))
  vocab.insert(0, '<PAD>')
  vocab_size = len(vocab)

  # Create an index mapping for the vocabulary.
  word2idx = {c:i for i, c in enumerate(vocab)}

  # convert stories from strings to lists of integers
  inputs_train, queries_train, answers_train = vectorize_stories(
    train_stories, 
    word2idx,
    story_maxlen,
    query_maxlen
  )
  inputs_test, queries_test, answers_test = vectorize_stories(
    test_stories, 
    word2idx,
    story_maxlen,
    query_maxlen
  )

  # convert inputs into 3-D numpy arrays
  inputs_train = stack_inputs(inputs_train, story_maxsents, story_maxlen)
  inputs_test = stack_inputs(inputs_test, story_maxsents, story_maxlen)
  print("inputs_train.shape, inputs_test.shape", inputs_train.shape, inputs_test.shape)


  # return model inputs for keras
  return train_stories, test_stories, \
    inputs_train, queries_train, answers_train, \
    inputs_test, queries_test, answers_test, \
    story_maxsents, story_maxlen, query_maxlen, \
    vocab, vocab_size