| 27 | |
| 28 | |
| 29 | class BufferedTokenStream(TokenStream): |
| 30 | __slots__ = ('tokenSource', 'tokens', 'index', 'fetchedEOF') |
| 31 | |
| 32 | def __init__(self, tokenSource:Lexer): |
| 33 | # The {@link TokenSource} from which tokens for this stream are fetched. |
| 34 | self.tokenSource = tokenSource |
| 35 | |
| 36 | # A collection of all tokens fetched from the token source. The list is |
| 37 | # considered a complete view of the input once {@link #fetchedEOF} is set |
| 38 | # to {@code true}. |
| 39 | self.tokens = [] |
| 40 | |
| 41 | # The index into {@link #tokens} of the current token (next token to |
| 42 | # {@link #consume}). {@link #tokens}{@code [}{@link #p}{@code ]} should be |
| 43 | # {@link #LT LT(1)}. |
| 44 | # |
| 45 | # <p>This field is set to -1 when the stream is first constructed or when |
| 46 | # {@link #setTokenSource} is called, indicating that the first token has |
| 47 | # not yet been fetched from the token source. For additional information, |
| 48 | # see the documentation of {@link IntStream} for a description of |
| 49 | # Initializing Methods.</p> |
| 50 | self.index = -1 |
| 51 | |
| 52 | # Indicates whether the {@link Token#EOF} token has been fetched from |
| 53 | # {@link #tokenSource} and added to {@link #tokens}. This field improves |
| 54 | # performance for the following cases: |
| 55 | # |
| 56 | # <ul> |
| 57 | # <li>{@link #consume}: The lookahead check in {@link #consume} to prevent |
| 58 | # consuming the EOF symbol is optimized by checking the values of |
| 59 | # {@link #fetchedEOF} and {@link #p} instead of calling {@link #LA}.</li> |
| 60 | # <li>{@link #fetch}: The check to prevent adding multiple EOF symbols into |
| 61 | # {@link #tokens} is trivial with this field.</li> |
| 62 | # <ul> |
| 63 | self.fetchedEOF = False |
| 64 | |
| 65 | def mark(self): |
| 66 | return 0 |
| 67 | |
| 68 | def release(self, marker:int): |
| 69 | # no resources to release |
| 70 | pass |
| 71 | |
| 72 | def reset(self): |
| 73 | self.seek(0) |
| 74 | |
| 75 | def seek(self, index:int): |
| 76 | self.lazyInit() |
| 77 | self.index = self.adjustSeekIndex(index) |
| 78 | |
| 79 | def get(self, index:int): |
| 80 | self.lazyInit() |
| 81 | return self.tokens[index] |
| 82 | |
| 83 | def consume(self): |
| 84 | skipEofCheck = False |
| 85 | if self.index >= 0: |
| 86 | if self.fetchedEOF: |
nothing calls this directly
no outgoing calls
no test coverage detected
searching dependent graphs…