(content)
| 5 | |
| 6 | // Extract User-agent / Allow / Disallow rules from a robots.txt body |
| 7 | const parseRobotsTxt = (content) => { |
| 8 | const rules = []; |
| 9 | for (let line of content.split('\n')) { |
| 10 | line = line.trim(); |
| 11 | const ruleMatch = line.match(/^(Allow|Disallow|User-agent):\s*(\S*)$/i); |
| 12 | if (ruleMatch) rules.push({ lbl: ruleMatch[1], val: ruleMatch[2] }); |
| 13 | } |
| 14 | return { robots: rules }; |
| 15 | }; |
| 16 | |
| 17 | const robotsHandler = async (url) => { |
| 18 | const { protocol, hostname } = parseTarget(url); |