Function process_data

_deeplearn_utils/dl_text/dl.py:56–142

(sent_l,sent_r=None,wordVec_model=None,dimx=100,dimy=100,vocab_size=10000,embedding_dim=300)

    return [x.strip() for x in re.split('(\W+)?', sent) if x.strip()]

def process_data(sent_l,sent_r=None,wordVec_model=None,dimx=100,dimy=100,vocab_size=10000,embedding_dim=300):
    sent1 = []
    sent1.extend(sent_l)
    if sent_r:
        sent1.extend(sent_r)
#    sent1 = [' '.join(i) for i in sent1]
    sentence = ["%s %s %s" % (START,x,END) for x in sent1]
    tokenize_sent = [regexp_tokenize(x, 
                                     pattern = '\w+|$[\d\.]+|\S+') for x in sentence]
    
    
    freq = FreqDist(chain(*tokenize_sent))
    print 'found ',len(freq),' unique words'
    vocab = freq.most_common(vocab_size - 1)
    index_to_word = [x[0] for x in vocab]
    index_to_word.append(unk_token)
    
    word_to_index = dict([(w,i) for i,w in enumerate(index_to_word)])
    
    for i,sent in enumerate(tokenize_sent):
        tokenize_sent[i] = [w if w in word_to_index else unk_token for w in sent]
    
    len_train = len(sent_l)
    text=[]
    for i in tokenize_sent:
        text.extend(i)
    
    sentences_x = []
    sentences_y = []
    
    for sent in tokenize_sent[0:len_train]:
        temp = [START for i in range(dimx)]
        for ind,word in enumerate(sent[0:dimx]):
            temp[ind] = word
        sentences_x.append(temp)
            
    X_data = []
    for i in sentences_x:
        temp = []
        for j in i:
            temp.append(word_to_index[j])
        temp = np.array(temp).T
        X_data.append(temp)
    
    X_data = np.array(X_data)
    
    if sent_r:
        for sent in tokenize_sent[len_train:]:
            temp = [START for i in range(dimy)]
            for ind,word in enumerate(sent[0:dimy]):
                temp[ind] = word       
            sentences_y.append(temp)

        y_data=[]
        for i in sentences_y:
            temp = []
            for j in i:
                temp.append(word_to_index[j])