Make tensorflow dataset for predicting similarity of testset samples with Simaese DNN Parameters: - source -- path to directory with source code files to analyze similarity - test -- path to the testsetrfile specifying pairs of so
(source, test, tokenizer)
| 99 | return tokens |
| 100 | |
| 101 | def makeDataset(source, test, tokenizer): |
| 102 | """ |
| 103 | Make tensorflow dataset |
| 104 | for predicting similarity of testset samples with Simaese DNN |
| 105 | Parameters: |
| 106 | - source -- path to directory with source code files |
| 107 | to analyze similarity |
| 108 | - test -- path to the testsetrfile specifying pairs |
| 109 | of source code file to analyze similarity |
| 110 | - tokenizer -- path to tokenizer executable |
| 111 | Returns: |
| 112 | - dataset as list of two numpy arrays. |
| 113 | Each numpy array represets set of token sequences for one input of DNN |
| 114 | """ |
| 115 | tokenizations = {} |
| 116 | samples = [] |
| 117 | max_code_len = 0 |
| 118 | with open(test, newline='') as csvfile: |
| 119 | test_reader = csv.reader(csvfile) |
| 120 | test_reader.__next__() #Skip csv header |
| 121 | for _num, fn1, fn2 in test_reader: |
| 122 | try: |
| 123 | tok_seq1 = tokenizations[fn1] |
| 124 | except KeyError: |
| 125 | tok_seq1 = tokenizeFile(source + '/' + fn1, tokenizer) |
| 126 | tokenizations[fn1] = tok_seq1 |
| 127 | max_code_len = max(max_code_len, len(tok_seq1)) |
| 128 | try: |
| 129 | tok_seq2 = tokenizations[fn2] |
| 130 | except KeyError: |
| 131 | tok_seq2 = tokenizeFile(source + '/' + fn2, tokenizer) |
| 132 | tokenizations[fn2] = tok_seq2 |
| 133 | max_code_len = max(max_code_len, len(tok_seq2)) |
| 134 | samples.append((tok_seq1, tok_seq2)) |
| 135 | np_ds1 = np.zeros(shape=(len(samples), max_code_len), |
| 136 | dtype=np.int32) |
| 137 | np_ds2 = np.zeros(shape=(len(samples), max_code_len), |
| 138 | dtype=np.int32) |
| 139 | for _i, _s in enumerate(samples): |
| 140 | tok_seq1, tok_seq2 = _s |
| 141 | np_ds1[_i][0:len(tok_seq1)] = np.asarray(tok_seq1, dtype=np.int32) |
| 142 | np_ds2[_i][0:len(tok_seq2)] = np.asarray(tok_seq2, dtype=np.int32) |
| 143 | print(f"Dataset of {len(samples)} samples is constructed") |
| 144 | return [np_ds1, np_ds2] |
| 145 | |
| 146 | def loadLabels(filename): |
| 147 | """ |