Make a token set The set of tokens must be the same as it used for trainning the DNN Here we make a CPP56X token set of C++ tokens Returns a dictionary of tokens: - Key is a string representing the token - Value is integer value of token
()
| 29 | import tensorflow as tf |
| 30 | |
| 31 | def makeTokenSet(): |
| 32 | """ |
| 33 | Make a token set |
| 34 | The set of tokens must be the same as it used for trainning the DNN |
| 35 | Here we make a CPP56X token set of C++ tokens |
| 36 | Returns a dictionary of tokens: |
| 37 | - Key is a string representing the token |
| 38 | - Value is integer value of token |
| 39 | """ |
| 40 | #CPP56 OPERATORS |
| 41 | operators = [ |
| 42 | "=", "+", "-", "*", "/", #Assignment and arithmetic operators |
| 43 | "%", "&", "|", "^", "~", "<<", ">>", #Bitwise Operators |
| 44 | "+=", "-=", "*=", "/=", "%=", "++", "--", #Compound arithmetic assignment operators |
| 45 | "&=", "|=", "^=", "<<=", ">>=", #Compound bitwise assignment operators |
| 46 | "==", "!=", "<", "<=", ">", ">=", #Comparison operators |
| 47 | "?", "&&", "||", "!", #Logical operators |
| 48 | "(", ")", "{", "}", "[", "]", "->", |
| 49 | ";", ","] #Others |
| 50 | #CPP56 KEYWORDS |
| 51 | keywords= ["if", "else", "for", "while", |
| 52 | "switch", |
| 53 | "enum", "int", "char", "short", "long", |
| 54 | "float", "double", "bool"] |
| 55 | #CPP SYNONYMS |
| 56 | synonyms = {"and": "&&", "or": "||", "not": "!"} |
| 57 | |
| 58 | token_dict = {} |
| 59 | for _i, _op in enumerate(operators + keywords): |
| 60 | token_dict[_op] = _i |
| 61 | print(f"Token set of {len(token_dict)} tokens is constructed") |
| 62 | for _syn, _orig in synonyms.items(): |
| 63 | token_dict[_syn] = token_dict[_orig] |
| 64 | print(f"Additionally it has {len(synonyms)} synonym tokens") |
| 65 | return token_dict |
| 66 | |
| 67 | #Dictionary of tokens and their indicies |
| 68 | token_set = makeTokenSet() |