| 105 | } |
| 106 | |
| 107 | protected void map(LongWritable key, Text value, Context context) |
| 108 | throws IOException, InterruptedException { |
| 109 | String url = value.toString().trim(); // value is line of text |
| 110 | |
| 111 | if (url != null && (url.length() == 0 || url.startsWith("#"))) { |
| 112 | /* Ignore line that start with # */ |
| 113 | return; |
| 114 | } |
| 115 | |
| 116 | // if tabs : metadata that could be stored |
| 117 | // must be name=value and separated by \t |
| 118 | float customScore = -1f; |
| 119 | int customInterval = interval; |
| 120 | Map<String, String> metadata = new TreeMap<String, String>(); |
| 121 | InjectType injectType = InjectType.INJECT; |
| 122 | if (url.indexOf("\t") != -1) { |
| 123 | String[] splits = url.split("\t"); |
| 124 | url = splits[0]; |
| 125 | for (int s = 1; s < splits.length; s++) { |
| 126 | // find separation between name and value |
| 127 | int indexEquals = splits[s].indexOf("="); |
| 128 | if (splits[s].indexOf("sitemaps:") > -1) { |
| 129 | String[] sitemaps = splits[s].trim().split(" "); |
| 130 | String sitemapUrl; |
| 131 | for (int i = 1; i < sitemaps.length; i++) { |
| 132 | sitemapUrl = url + sitemaps[i]; |
| 133 | write(sitemapUrl, context, customInterval, customScore, |
| 134 | new HashMap<String, String>(), InjectType.SITEMAP_INJECT); |
| 135 | } |
| 136 | continue; |
| 137 | } else if (splits[s].indexOf("-sitemap") == 0) { |
| 138 | injectType = InjectType.SITEMAP_INJECT; |
| 139 | continue; |
| 140 | } else if (indexEquals == -1) { |
| 141 | // skip anything without a = |
| 142 | continue; |
| 143 | } |
| 144 | String metaname = splits[s].substring(0, indexEquals); |
| 145 | String metavalue = splits[s].substring(indexEquals + 1); |
| 146 | if (metaname.equals(nutchScoreMDName)) { |
| 147 | try { |
| 148 | customScore = Float.parseFloat(metavalue); |
| 149 | } catch (NumberFormatException nfe) { |
| 150 | } |
| 151 | } else if (metaname.equals(nutchFetchIntervalMDName)) { |
| 152 | try { |
| 153 | customInterval = Integer.parseInt(metavalue); |
| 154 | } catch (NumberFormatException nfe) { |
| 155 | } |
| 156 | } else |
| 157 | metadata.put(metaname, metavalue); |
| 158 | } |
| 159 | } |
| 160 | write(url, context, customInterval, customScore, metadata, injectType); |
| 161 | } |
| 162 | |
| 163 | private void write(String url, Context context, Integer customInterval, |
| 164 | Float customScore, Map<String, String> metadata, InjectType injectType) |