Iterate through all the items in this structured DMOZ file. Add each URL to the web db.
(File dmozFile, int subsetDenom,
boolean includeAdult, int skew, Pattern topicPattern, boolean snippet)
| 304 | * the web db. |
| 305 | */ |
| 306 | public void parseDmozFile(File dmozFile, int subsetDenom, |
| 307 | boolean includeAdult, int skew, Pattern topicPattern, boolean snippet) |
| 308 | |
| 309 | throws IOException, SAXException, ParserConfigurationException { |
| 310 | |
| 311 | SAXParserFactory parserFactory = SAXParserFactory.newInstance(); |
| 312 | SAXParser parser = parserFactory.newSAXParser(); |
| 313 | XMLReader reader = parser.getXMLReader(); |
| 314 | |
| 315 | // Create our own processor to receive SAX events |
| 316 | RDFProcessor rp = new RDFProcessor(reader, subsetDenom, includeAdult, skew, |
| 317 | topicPattern, snippet); |
| 318 | reader.setContentHandler(rp); |
| 319 | reader.setErrorHandler(rp); |
| 320 | LOG.info("skew = " + rp.hashSkew); |
| 321 | |
| 322 | // |
| 323 | // Open filtered text stream. The TextFilter makes sure that |
| 324 | // only appropriate XML-approved Text characters are received. |
| 325 | // Any non-conforming characters are silently skipped. |
| 326 | // |
| 327 | XMLCharFilter in = new XMLCharFilter(new BufferedReader( |
| 328 | new InputStreamReader(new BufferedInputStream(new FileInputStream( |
| 329 | dmozFile)), "UTF-8"))); |
| 330 | try { |
| 331 | InputSource is = new InputSource(in); |
| 332 | reader.parse(is); |
| 333 | } catch (Exception e) { |
| 334 | if (LOG.isErrorEnabled()) { |
| 335 | LOG.error("Failed to retieve the InputSource: ", e.toString()); |
| 336 | } |
| 337 | System.exit(0); |
| 338 | } finally { |
| 339 | in.close(); |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | private static void addTopicsFromFile(String topicFile, Vector<String> topics) |
| 344 | throws IOException { |