MCPcopy Create free account
hub / github.com/apache/nutch / read

Method read

src/java/org/apache/nutch/crawl/WebTableReader.java:227–260  ·  view source on GitHub ↗

Prints out the entry to the standard out

(String key, boolean dumpContent, boolean dumpHeaders,
      boolean dumpLinks, boolean dumpText)

Source from the content-addressed store, hash-verified

225
226 /** Prints out the entry to the standard out **/
227 private void read(String key, boolean dumpContent, boolean dumpHeaders,
228 boolean dumpLinks, boolean dumpText) throws ClassNotFoundException,
229 IOException, Exception {
230 DataStore<String, WebPage> datastore = StorageUtils.createWebStore(
231 getConf(), String.class, WebPage.class);
232
233 Query<String, WebPage> query = datastore.newQuery();
234 String reversedUrl = TableUtil.reverseUrl(key);
235 query.setKey(reversedUrl);
236
237 Result<String, WebPage> result = datastore.execute(query);
238 boolean found = false;
239 // should happen only once
240 while (result.next()) {
241 try {
242 WebPage page = result.get();
243 String skey = result.getKey();
244 // we should not get to this point but nevermind
245 if (page == null || skey == null)
246 break;
247 found = true;
248 String url = TableUtil.unreverseUrl(skey);
249 System.out.println(getPageRepresentation(url, page, dumpContent,
250 dumpHeaders, dumpLinks, dumpText));
251 } catch (Exception e) {
252 LOG.error(StringUtils.stringifyException(e));
253 }
254 }
255 if (!found) {
256 LOG.info("{} not found", key);
257 }
258 result.close();
259 datastore.close();
260 }
261
262 /** Filters the entries from the table based on a regex **/
263 public static class WebTableRegexMapper extends

Callers 1

runMethod · 0.95

Calls 10

createWebStoreMethod · 0.95
reverseUrlMethod · 0.95
unreverseUrlMethod · 0.95
getPageRepresentationMethod · 0.95
getKeyMethod · 0.80
errorMethod · 0.80
getMethod · 0.65
closeMethod · 0.65
getConfMethod · 0.45
nextMethod · 0.45

Tested by

no test coverage detected