| 26 | r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~\'"@]*))?') # this is taken from sgmllib |
| 27 | |
| 28 | class Scraper: |
| 29 | def __init__(self): |
| 30 | """Initialise a parser.""" |
| 31 | self.buffer = '' |
| 32 | self.outfile = '' |
| 33 | |
| 34 | def reset(self): |
| 35 | """This method clears the input buffer and the output buffer.""" |
| 36 | self.buffer = '' |
| 37 | self.outfile = '' |
| 38 | |
| 39 | def push(self): |
| 40 | """This returns all currently processed data and empties the output buffer.""" |
| 41 | data = self.outfile |
| 42 | self.outfile = '' |
| 43 | return data |
| 44 | |
| 45 | def close(self): |
| 46 | """Returns any unprocessed data (without processing it) and resets the parser. |
| 47 | Should be used after all the data has been handled using feed and then collected with push. |
| 48 | This returns any trailing data that can't be processed. |
| 49 | |
| 50 | If you are processing everything in one go you can safely use this method to return everything. |
| 51 | """ |
| 52 | data = self.push() + self.buffer |
| 53 | self.buffer = '' |
| 54 | return data |
| 55 | |
| 56 | def feed(self, data): |
| 57 | """Pass more data into the parser. |
| 58 | As much as possible is processed - but nothing is returned from this method. |
| 59 | """ |
| 60 | self.index = -1 |
| 61 | self.tempindex = 0 |
| 62 | self.buffer = self.buffer + data |
| 63 | outlist = [] |
| 64 | thischunk = [] |
| 65 | while self.index < len(self.buffer)-1: # rewrite with a list of all the occurences of '<' and jump between them, much faster than character by character - which is fast enough to be fair... |
| 66 | self.index += 1 |
| 67 | inchar = self.buffer[self.index] |
| 68 | if inchar == '<': |
| 69 | outlist.append(self.pdata(''.join(thischunk))) |
| 70 | thischunk = [] |
| 71 | result = self.tagstart() |
| 72 | if result: outlist.append(result) |
| 73 | if self.tempindex: break |
| 74 | else: |
| 75 | thischunk.append(inchar) |
| 76 | if self.tempindex: |
| 77 | self.buffer = self.buffer[self.tempindex:] |
| 78 | else: |
| 79 | self.buffer = '' |
| 80 | if thischunk: self.buffer = ''.join(thischunk) |
| 81 | self.outfile = self.outfile + ''.join(outlist) |
| 82 | |
| 83 | def tagstart(self): |
| 84 | """We have reached the start of a tag. |
| 85 | self.buffer is the data |