(currentURL: string, limit: number)
| 506 | } |
| 507 | |
| 508 | export async function xmlScrape(currentURL: string, limit: number): Promise<string[]> { |
| 509 | let urls: string[] = [] |
| 510 | if (process.env.DEBUG === 'true') console.info(`actively scarping ${currentURL}`) |
| 511 | try { |
| 512 | const resp = await secureFetch(currentURL) |
| 513 | |
| 514 | if (resp.status > 399) { |
| 515 | if (process.env.DEBUG === 'true') console.error(`error in fetch with status code: ${resp.status}, on page: ${currentURL}`) |
| 516 | return urls |
| 517 | } |
| 518 | |
| 519 | const contentType: string | null = resp.headers.get('content-type') |
| 520 | if ((contentType && !contentType.includes('application/xml') && !contentType.includes('text/xml')) || !contentType) { |
| 521 | if (process.env.DEBUG === 'true') console.error(`non xml response, content type: ${contentType}, on page: ${currentURL}`) |
| 522 | return urls |
| 523 | } |
| 524 | |
| 525 | const xmlBody = await resp.text() |
| 526 | urls = getURLsFromXML(xmlBody, limit) |
| 527 | } catch (err) { |
| 528 | if (process.env.DEBUG === 'true') console.error(`error in fetch url: ${err.message}, on page: ${currentURL}`) |
| 529 | } |
| 530 | return urls |
| 531 | } |
| 532 | |
| 533 | /** |
| 534 | * Get env variables |
no test coverage detected