(String url, Context context, Integer customInterval,
Float customScore, Map<String, String> metadata, InjectType injectType)
| 161 | } |
| 162 | |
| 163 | private void write(String url, Context context, Integer customInterval, |
| 164 | Float customScore, Map<String, String> metadata, InjectType injectType) |
| 165 | throws IOException, InterruptedException { |
| 166 | try { |
| 167 | url = urlNormalizers.normalize(url, URLNormalizers.SCOPE_INJECT); |
| 168 | url = filters.filter(url); // filter the url |
| 169 | } catch (Exception e) { |
| 170 | LOG.warn("Skipping " + url + ":" + e); |
| 171 | url = null; |
| 172 | } |
| 173 | if (url == null) { |
| 174 | context.getCounter("injector", "urls_filtered").increment(1); |
| 175 | return; |
| 176 | } else { // if it passes |
| 177 | String reversedUrl = TableUtil.reverseUrl(url); // collect it |
| 178 | WebPage row = WebPage.newBuilder().build(); |
| 179 | row.setFetchTime(curTime); |
| 180 | row.setFetchInterval(customInterval); |
| 181 | |
| 182 | // now add the metadata |
| 183 | Iterator<String> keysIter = metadata.keySet().iterator(); |
| 184 | while (keysIter.hasNext()) { |
| 185 | String keymd = keysIter.next(); |
| 186 | String valuemd = metadata.get(keymd); |
| 187 | row.getMetadata().put(new Utf8(keymd), |
| 188 | ByteBuffer.wrap(valuemd.getBytes(StandardCharsets.UTF_8))); |
| 189 | } |
| 190 | |
| 191 | if (customScore != -1) |
| 192 | row.setScore(customScore); |
| 193 | else |
| 194 | row.setScore(scoreInjected); |
| 195 | |
| 196 | try { |
| 197 | scfilters.injectedScore(url, row); |
| 198 | } catch (ScoringFilterException e) { |
| 199 | if (LOG.isWarnEnabled()) { |
| 200 | LOG.warn("Cannot filter injected score for url {}, using default ({})", url, e.getMessage()); |
| 201 | } |
| 202 | } |
| 203 | context.getCounter("injector", "urls_injected").increment(1); |
| 204 | row.getMarkers() |
| 205 | .put(DbUpdaterJob.DISTANCE, new Utf8(String.valueOf(0))); |
| 206 | Mark.INJECT_MARK.putMark(row, injectType.getTypeString()); |
| 207 | context.write(reversedUrl, row); |
| 208 | } |
| 209 | } |
| 210 | } |
| 211 | |
| 212 | public InjectorJob() { |
no test coverage detected