Class MapReduceScore

src/com/winvector/logistic/demo/MapReduceScore.java:29–102

/// @SuppressWarnings("deprecation") //  Tool depreciated 0.21.0 a known issue: https://issues.apache.org/jira/browse/MAPREDUCE-2084
public class MapReduceScore extends Configured implements Tool {

	@Override
	public int run(final String[] args) throws Exception {		
		if(args.length!=3) {
			final Log log = LogFactory.getLog(MapReduceScore.class);
			log.info(Licenses.licenses);
			log.fatal("use: MapReduceScore model.ser testFile resultDir");
			return -1;
		}
		final String modelFileName = args[0];
		final String testFileName = args[1];
		final String resultFileName = args[2];
		run(modelFileName,testFileName,resultFileName);
		return 0;
	}	
		
	public double run(final String modelFileName, final String testFileName, final String resultFileName) throws Exception {
		final Log log = LogFactory.getLog(MapReduceScore.class);
		final Random rand = new Random();
		final String tmpPrefix = "TMPAC_" + rand.nextLong();
		final Configuration mrConfig = getConf();
		log.info("start");
		log.info("reading model: " + modelFileName);
		final Model model;
		{
			final Path modelPath = new Path(modelFileName);
			final FSDataInputStream fdi = modelPath.getFileSystem(mrConfig).open(modelPath);
			final ObjectInputStream ois = new ObjectInputStream(fdi);
			model = (Model)ois.readObject();		
			ois.close();
		}
		log.info("model:\n" + model.config.formatSoln(model.coefs));
		final Path testFile = new Path(testFileName);
		final Path resultFile = new Path(resultFileName);
		log.info("scoring data: " + testFile);
		log.info("writing: " + resultFile);
		final SigmoidLossMultinomial underlying = new SigmoidLossMultinomial(model.config.dim(),model.config.noutcomes());
		final WritableVariableList lConfig = WritableVariableList.copy(model.config.def());
		final String headerLine = WritableUtils.readFirstLine(mrConfig,testFile);
		final Pattern sepPattern = Pattern.compile("\t");
		final LineBurster burster = new HBurster(sepPattern,headerLine,false);
		mrConfig.set(MapRedScan.BURSTERSERFIELD,SerialUtils.serializableToString(burster));
		final StringBuilder b = new StringBuilder();
		b.append("predict" + "." + model.config.def().resultColumn + "\t");
		b.append("predict" + "." + model.config.def().resultColumn + "." + "score" + "\t");
		for(int i=0;i<model.config.noutcomes();++i) {
			final String cat = model.config.outcome(i);
			b.append("predict" + "." + model.config.def().resultColumn + "." + cat + "." + "score" + "\t");
		}
		b.append(headerLine);
		mrConfig.set(MapRedScore.IDEALHEADERFIELD,b.toString());
		final MapRedScore sc = new MapRedScore(underlying,lConfig,model.config.useIntercept(),mrConfig,testFile);				
		sc.score(model.coefs,resultFile);
		final MapRedAccuracy ac = new MapRedAccuracy(underlying,lConfig,model.config.useIntercept(),tmpPrefix,mrConfig,testFile);
		final long[] testAccuracy = ac.score(model.coefs);
		final double accuracy = testAccuracy[0]/(double)testAccuracy[1];
		log.info("test accuracy: " + testAccuracy[0] + "/" + testAccuracy[1] + "\t" + accuracy);