| 16 | from pos_baseline import LogisticRegression |
| 17 | |
| 18 | def get_data(split_sequences=False): |
| 19 | word2idx = {} |
| 20 | tag2idx = {} |
| 21 | word_idx = 0 |
| 22 | tag_idx = 0 |
| 23 | Xtrain = [] |
| 24 | Ytrain = [] |
| 25 | currentX = [] |
| 26 | currentY = [] |
| 27 | for line in open('ner.txt'): |
| 28 | line = line.rstrip() |
| 29 | if line: |
| 30 | r = line.split() |
| 31 | word, tag = r |
| 32 | word = word.lower() |
| 33 | if word not in word2idx: |
| 34 | word2idx[word] = word_idx |
| 35 | word_idx += 1 |
| 36 | currentX.append(word2idx[word]) |
| 37 | |
| 38 | if tag not in tag2idx: |
| 39 | tag2idx[tag] = tag_idx |
| 40 | tag_idx += 1 |
| 41 | currentY.append(tag2idx[tag]) |
| 42 | elif split_sequences: |
| 43 | Xtrain.append(currentX) |
| 44 | Ytrain.append(currentY) |
| 45 | currentX = [] |
| 46 | currentY = [] |
| 47 | |
| 48 | if not split_sequences: |
| 49 | Xtrain = currentX |
| 50 | Ytrain = currentY |
| 51 | |
| 52 | print("number of samples:", len(Xtrain)) |
| 53 | Xtrain, Ytrain = shuffle(Xtrain, Ytrain) |
| 54 | Ntest = int(0.3*len(Xtrain)) |
| 55 | Xtest = Xtrain[:Ntest] |
| 56 | Ytest = Ytrain[:Ntest] |
| 57 | Xtrain = Xtrain[Ntest:] |
| 58 | Ytrain = Ytrain[Ntest:] |
| 59 | print("number of classes:", len(tag2idx)) |
| 60 | return Xtrain, Ytrain, Xtest, Ytest, word2idx, tag2idx |
| 61 | |
| 62 | |
| 63 | # def get_data2(split_sequences=False): |