(html_file)
| 297 | return filter_and_update_toc(toc) |
| 298 | |
| 299 | def parse_html(html_file): |
| 300 | soup = bs4.BeautifulSoup(html_file, "lxml") |
| 301 | # parse title |
| 302 | title = soup.head.title.get_text().replace("\n", " ") |
| 303 | # parse abstract |
| 304 | abstract = soup.find(class_='ltx_abstract').get_text() |
| 305 | # parse citation |
| 306 | citation = soup.find(class_='ltx_biblist') |
| 307 | citation_dict = create_dict_for_citation(citation) |
| 308 | # generate the full toc without text |
| 309 | sections = generate_full_toc(soup) |
| 310 | # remove the sections need to skip and extract the text of the rest sections |
| 311 | sections = remove_stop_word_sections_and_extract_text(sections, soup) |
| 312 | document = { |
| 313 | "title": title, |
| 314 | "abstract": abstract, |
| 315 | "sections": sections, |
| 316 | "references": citation_dict, |
| 317 | } |
| 318 | return document |
| 319 | |
| 320 | def search_section_by_arxiv_id(entry_id, cite): |
| 321 | warnings.warn("Using search_section_by_arxiv_id function may return wrong title because ar5iv parsing citation error. To solve this, You can prompt any LLM to extract the paper title from the reference string") |
no test coverage detected