Tokenize a given file Parameters: - filename -- name of source code file to tokenize - tokenizer -- path to tokenizer executable Returns: - a list of integer token values representing the source code file
(filename, tokenizer)
| 68 | token_set = makeTokenSet() |
| 69 | |
| 70 | def tokenizeFile(filename, tokenizer): |
| 71 | """ |
| 72 | Tokenize a given file |
| 73 | Parameters: |
| 74 | - filename -- name of source code file to tokenize |
| 75 | - tokenizer -- path to tokenizer executable |
| 76 | Returns: |
| 77 | - a list of integer token values representing the source code file |
| 78 | """ |
| 79 | #Name of temporary file for tokenized source code |
| 80 | TMP_TOKENIZATION = "./t_o_k_e_n_s.o_u_t" |
| 81 | #Tokenization command ignoring macros |
| 82 | #tokenize_cmd = tokenizer + " -wcmcsv" |
| 83 | #Tokenization command tokenising macros |
| 84 | tokenize_cmd = tokenizer + " -wmcsv" |
| 85 | if os.system(f"{tokenize_cmd} -o {TMP_TOKENIZATION} {filename}"): |
| 86 | sys.exit(f"Tokenization error in file {filename}") |
| 87 | tokens = [] |
| 88 | with open(TMP_TOKENIZATION, newline='', |
| 89 | encoding="ISO-8859-1") as csvfile: |
| 90 | token_reader = csv.reader(csvfile) |
| 91 | token_reader.__next__() #Skip csv header |
| 92 | for _, _, _tok_class, _tok_value in token_reader: |
| 93 | if _tok_class == "operator" or _tok_class == "keyword": |
| 94 | try: |
| 95 | tokens.append(token_set[_tok_value] + 1) |
| 96 | except KeyError: |
| 97 | #ignore tokens that are not in the tokens set |
| 98 | pass |
| 99 | return tokens |
| 100 | |
| 101 | def makeDataset(source, test, tokenizer): |
| 102 | """ |