MCPcopy Create free account
hub / github.com/apache/nutch / write

Method write

src/java/org/apache/nutch/crawl/InjectorJob.java:163–209  ·  view source on GitHub ↗
(String url, Context context, Integer customInterval,
        Float customScore, Map<String, String> metadata, InjectType injectType)

Source from the content-addressed store, hash-verified

161 }
162
163 private void write(String url, Context context, Integer customInterval,
164 Float customScore, Map<String, String> metadata, InjectType injectType)
165 throws IOException, InterruptedException {
166 try {
167 url = urlNormalizers.normalize(url, URLNormalizers.SCOPE_INJECT);
168 url = filters.filter(url); // filter the url
169 } catch (Exception e) {
170 LOG.warn("Skipping " + url + ":" + e);
171 url = null;
172 }
173 if (url == null) {
174 context.getCounter("injector", "urls_filtered").increment(1);
175 return;
176 } else { // if it passes
177 String reversedUrl = TableUtil.reverseUrl(url); // collect it
178 WebPage row = WebPage.newBuilder().build();
179 row.setFetchTime(curTime);
180 row.setFetchInterval(customInterval);
181
182 // now add the metadata
183 Iterator<String> keysIter = metadata.keySet().iterator();
184 while (keysIter.hasNext()) {
185 String keymd = keysIter.next();
186 String valuemd = metadata.get(keymd);
187 row.getMetadata().put(new Utf8(keymd),
188 ByteBuffer.wrap(valuemd.getBytes(StandardCharsets.UTF_8)));
189 }
190
191 if (customScore != -1)
192 row.setScore(customScore);
193 else
194 row.setScore(scoreInjected);
195
196 try {
197 scfilters.injectedScore(url, row);
198 } catch (ScoringFilterException e) {
199 if (LOG.isWarnEnabled()) {
200 LOG.warn("Cannot filter injected score for url {}, using default ({})", url, e.getMessage());
201 }
202 }
203 context.getCounter("injector", "urls_injected").increment(1);
204 row.getMarkers()
205 .put(DbUpdaterJob.DISTANCE, new Utf8(String.valueOf(0)));
206 Mark.INJECT_MARK.putMark(row, injectType.getTypeString());
207 context.write(reversedUrl, row);
208 }
209 }
210 }
211
212 public InjectorJob() {

Callers 1

mapMethod · 0.95

Calls 15

reverseUrlMethod · 0.95
newBuilderMethod · 0.95
setFetchTimeMethod · 0.95
setFetchIntervalMethod · 0.95
getMetadataMethod · 0.95
setScoreMethod · 0.95
getMarkersMethod · 0.95
putMarkMethod · 0.80
getTypeStringMethod · 0.80
normalizeMethod · 0.65
filterMethod · 0.65
getMethod · 0.65

Tested by

no test coverage detected