Function train_model

nlp_class2/word2vec.py:84–204

(savedir)

def train_model(savedir):
  # get the data
  sentences, word2idx = get_wiki() #get_brown()


  # number of unique words
  vocab_size = len(word2idx)


  # config
  window_size = 5
  learning_rate = 0.025
  final_learning_rate = 0.0001
  num_negatives = 5 # number of negative samples to draw per input word
  epochs = 20
  D = 50 # word embedding size


  # learning rate decay
  learning_rate_delta = (learning_rate - final_learning_rate) / epochs


  # params
  W = np.random.randn(vocab_size, D) # input-to-hidden
  V = np.random.randn(D, vocab_size) # hidden-to-output


  # distribution for drawing negative samples
  p_neg = get_negative_sampling_distribution(sentences, vocab_size)


  # save the costs to plot them per iteration
  costs = []


  # number of total words in corpus
  total_words = sum(len(sentence) for sentence in sentences)
  print("total number of words in corpus:", total_words)

  # for subsampling each sentence
  threshold = 1e-5
  p_drop = 1 - np.sqrt(threshold / p_neg)


  # train the model
  for epoch in range(epochs):
    # randomly order sentences so we don't always see
    # sentences in the same order
    np.random.shuffle(sentences)

    # accumulate the cost
    cost = 0
    counter = 0
    t0 = datetime.now()
    for sentence in sentences:
      # keep only certain words based on p_neg
      sentence = [w for w in sentence \
        if np.random.random() < (1 - p_drop[w])