| 6 | type ParseFunction = (html: string, baseUrl: string) => Package[]; |
| 7 | |
| 8 | export async function getPackagesFromCran(cranUrl: string): Promise<Package[]> { |
| 9 | const cranSites: {url: string, parseFunction: ParseFunction}[] = [ |
| 10 | // NOTE: Not working any more |
| 11 | // { |
| 12 | // url: new URL('stats/descriptions', cranUrl).toString(), |
| 13 | // parseFunction: parseCranJson |
| 14 | // }, |
| 15 | { |
| 16 | url: new URL('web/packages/available_packages_by_date.html', cranUrl).toString(), |
| 17 | parseFunction: parseCranTable |
| 18 | }, |
| 19 | { |
| 20 | url: new URL('src/contrib/PACKAGES', cranUrl).toString(), |
| 21 | parseFunction: parseCranPackagesFile |
| 22 | } |
| 23 | ]; |
| 24 | let packages: Package[] = []; |
| 25 | for(const site of cranSites){ |
| 26 | try{ |
| 27 | // fetch html |
| 28 | process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; // seems to fail otherwise? |
| 29 | const res = await fetch(site.url); |
| 30 | const html = await (res).text(); |
| 31 | |
| 32 | // parse html |
| 33 | packages = site.parseFunction(html, site.url); |
| 34 | } catch(e) { |
| 35 | // These errors are expected, if the repo does not serve a specific URL |
| 36 | } finally { |
| 37 | // make sure to use safe https again |
| 38 | process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1'; |
| 39 | } |
| 40 | |
| 41 | // break if successfully fetched & parsed |
| 42 | if(packages?.length){ |
| 43 | break; |
| 44 | } |
| 45 | } |
| 46 | return packages; |
| 47 | } |
| 48 | |
| 49 | function parseCranPackagesFile(html: string): Package[] { |
| 50 | const packageNames = html.match(/^Package: .*$/gm)?.map(s => s.replace(/^Package: /, '')) || []; |