| 131 | return raw |
| 132 | |
| 133 | def read_token(self): |
| 134 | quoted = False |
| 135 | escapedstate = ' ' |
| 136 | while True: |
| 137 | if self.punctuation_chars and self._pushback_chars: |
| 138 | nextchar = self._pushback_chars.pop() |
| 139 | else: |
| 140 | nextchar = self.instream.read(1) |
| 141 | if nextchar == '\n': |
| 142 | self.lineno += 1 |
| 143 | if self.debug >= 3: |
| 144 | print("shlex: in state %r I see character: %r" % (self.state, |
| 145 | nextchar)) |
| 146 | if self.state is None: |
| 147 | self.token = '' # past end of file |
| 148 | break |
| 149 | elif self.state == ' ': |
| 150 | if not nextchar: |
| 151 | self.state = None # end of file |
| 152 | break |
| 153 | elif nextchar in self.whitespace: |
| 154 | if self.debug >= 2: |
| 155 | print("shlex: I see whitespace in whitespace state") |
| 156 | if self.token or (self.posix and quoted): |
| 157 | break # emit current token |
| 158 | else: |
| 159 | continue |
| 160 | elif nextchar in self.commenters: |
| 161 | self.instream.readline() |
| 162 | self.lineno += 1 |
| 163 | elif self.posix and nextchar in self.escape: |
| 164 | escapedstate = 'a' |
| 165 | self.state = nextchar |
| 166 | elif nextchar in self.wordchars: |
| 167 | self.token = nextchar |
| 168 | self.state = 'a' |
| 169 | elif nextchar in self.punctuation_chars: |
| 170 | self.token = nextchar |
| 171 | self.state = 'c' |
| 172 | elif nextchar in self.quotes: |
| 173 | if not self.posix: |
| 174 | self.token = nextchar |
| 175 | self.state = nextchar |
| 176 | elif self.whitespace_split: |
| 177 | self.token = nextchar |
| 178 | self.state = 'a' |
| 179 | else: |
| 180 | self.token = nextchar |
| 181 | if self.token or (self.posix and quoted): |
| 182 | break # emit current token |
| 183 | else: |
| 184 | continue |
| 185 | elif self.state in self.quotes: |
| 186 | quoted = True |
| 187 | if not nextchar: # end of file |
| 188 | if self.debug >= 2: |
| 189 | print("shlex: I see EOF in quotes state") |
| 190 | # XXX what error should be raised here? |