MCPcopy Index your code
hub / github.com/nodejs/node / Tokenize

Method Tokenize

tools/test.py:1133–1171  ·  view source on GitHub ↗
(self)

Source from the content-addressed store, hash-verified

1131 self.Advance()
1132
1133 def Tokenize(self):
1134 self.tokens = [ ]
1135 while self.HasMore():
1136 self.SkipSpaces()
1137 if not self.HasMore():
1138 return None
1139 if self.Current() == '(':
1140 self.AddToken('(')
1141 self.Advance()
1142 elif self.Current() == ')':
1143 self.AddToken(')')
1144 self.Advance()
1145 elif self.Current() == '$':
1146 self.AddToken('$')
1147 self.Advance()
1148 elif self.Current() == ',':
1149 self.AddToken(',')
1150 self.Advance()
1151 elif IsAlpha(self.Current()):
1152 buf = ""
1153 while self.HasMore() and IsAlpha(self.Current()):
1154 buf += self.Current()
1155 self.Advance()
1156 self.AddToken(buf)
1157 elif self.Current(2) == '&&':
1158 self.AddToken('&&')
1159 self.Advance(2)
1160 elif self.Current(2) == '||':
1161 self.AddToken('||')
1162 self.Advance(2)
1163 elif self.Current(2) == '==':
1164 self.AddToken('==')
1165 self.Advance(2)
1166 elif self.Current(2) == '!=':
1167 self.AddToken('!=')
1168 self.Advance(2)
1169 else:
1170 return None
1171 return self.tokens
1172
1173
1174class Scanner(object):

Callers 1

ParseConditionFunction · 0.80

Calls 6

HasMoreMethod · 0.95
SkipSpacesMethod · 0.95
CurrentMethod · 0.95
AddTokenMethod · 0.95
AdvanceMethod · 0.95
IsAlphaFunction · 0.85

Tested by

no test coverage detected