| 220 | return toc |
| 221 | |
| 222 | def parse_text(local_text, tag): |
| 223 | ignore_tags = ['a', 'figure', 'center', 'caption', 'td', 'h1', 'h2', 'h3', 'h4'] |
| 224 | # latexmlc |
| 225 | ignore_tags += ['sup'] |
| 226 | max_math_length = 300000 |
| 227 | |
| 228 | for child in tag.children: |
| 229 | child_type = type(child) |
| 230 | if child_type == bs4.element.NavigableString: |
| 231 | txt = child.get_text() |
| 232 | local_text.append(txt) |
| 233 | |
| 234 | elif child_type == bs4.element.Comment: |
| 235 | continue |
| 236 | elif child_type == bs4.element.Tag: |
| 237 | |
| 238 | if child.name in ignore_tags or (child.has_attr('class') and child['class'][0] == 'navigation'): |
| 239 | continue |
| 240 | elif child.name == 'cite': |
| 241 | # add hrefs |
| 242 | hrefs = [a.get('href').strip('#') for a in child.find_all('a', class_='ltx_ref')] |
| 243 | local_text.append('~\cite{' + ', '.join(hrefs) + '}') |
| 244 | elif child.name == 'img' and child.has_attr('alt'): |
| 245 | math_txt = child.get('alt') |
| 246 | if len(math_txt) < max_math_length: |
| 247 | local_text.append(math_txt) |
| 248 | |
| 249 | elif child.has_attr('class') and (child['class'][0] == 'ltx_Math' or child['class'][0] == 'ltx_equation'): |
| 250 | math_txt = child.get_text() |
| 251 | if len(math_txt) < max_math_length: |
| 252 | local_text.append(math_txt) |
| 253 | |
| 254 | elif child.name == 'section': |
| 255 | return |
| 256 | else: |
| 257 | parse_text(local_text, child) |
| 258 | else: |
| 259 | raise RuntimeError('Unhandled type') |
| 260 | |
| 261 | def clean_text(text): |
| 262 | delete_items = ['=-1', '\t', u'\xa0', '[]', '()', 'mathbb', 'mathcal', 'bm', 'mathrm', 'mathit', 'mathbf', 'mathbfcal', 'textbf', 'textsc', 'langle', 'rangle', 'mathbin'] |