(final String tmpPrefix, final Configuration mrConfig, final WritableVariableList defs, final Path pathIn)
| 189 | |
| 190 | |
| 191 | private static WritableVariableList ruLevelStep(final String tmpPrefix, final Configuration mrConfig, |
| 192 | final WritableVariableList defs, final Path pathIn) throws IOException, InterruptedException, ClassNotFoundException { |
| 193 | // write side-channel information |
| 194 | mrConfig.set(DEFFIELD,defs.toString()); |
| 195 | // run the job |
| 196 | final Path pathOut = new Path(tmpPrefix + "_MPRedLevelOut"); |
| 197 | final Job job = WritableUtils.newJob(mrConfig); |
| 198 | job.setJarByClass(MapRedScan.class); |
| 199 | job.setJobName("MapRevLevelStep"); |
| 200 | FileInputFormat.setMaxInputSplitSize(job,4*1024*1024L); |
| 201 | FileInputFormat.setMinInputSplitSize(job,4*1024L); |
| 202 | FileInputFormat.addInputPath(job, pathIn); |
| 203 | FileOutputFormat.setOutputPath(job, pathOut); |
| 204 | job.setMapperClass(LevMapper.class); |
| 205 | job.setCombinerClass(LevReducer.class); |
| 206 | job.setReducerClass(LevReducer.class); |
| 207 | job.setOutputKeyClass(LongWritable.class); |
| 208 | job.setOutputValueClass(WritableVariableList.class); |
| 209 | job.setInputFormatClass(TextInputFormat.class); |
| 210 | job.setOutputFormatClass(TextOutputFormat.class); |
| 211 | job.setNumReduceTasks(1); |
| 212 | WritableVariableList r = null; |
| 213 | if(job.waitForCompletion(false)) { |
| 214 | // collect results |
| 215 | final FSDataInputStream fdi = pathOut.getFileSystem(mrConfig).open(new Path(pathOut,"part-r-00000")); |
| 216 | final BufferedReader d = new BufferedReader(new InputStreamReader(fdi)); |
| 217 | final String line = d.readLine(); |
| 218 | d.close(); |
| 219 | final String[] flds = line.split("\t",2); |
| 220 | r = WritableVariableList.fromString(flds[1]); |
| 221 | } |
| 222 | // clean up |
| 223 | pathOut.getFileSystem(mrConfig).delete(pathOut,true); |
| 224 | return r; |
| 225 | } |
| 226 | |
| 227 | |
| 228 | public static WritableVariableList initialScan(final String tmpPrefix, final Configuration mrConfig, |
no test coverage detected