| 159 | } |
| 160 | |
| 161 | public static BTable buildStatBasedEncodings(final Set<String> varsToEncode, |
| 162 | final Iterable<BurstMap> trainSource, final VariableEncodings oldAdapter, |
| 163 | final double[] oldX, final Random rand) { |
| 164 | final Log log = LogFactory.getLog(BTable.class); |
| 165 | final DModel<ExampleRow> sigmoidLoss; |
| 166 | if(oldX!=null) { |
| 167 | sigmoidLoss = new SigmoidLossMultinomial(oldAdapter.dim(),oldAdapter.noutcomes()); |
| 168 | } else { |
| 169 | sigmoidLoss = null; |
| 170 | } |
| 171 | // go through data to get stats |
| 172 | log.info("start variable re-encoding scan"); |
| 173 | final BSampler bsampler = new BSampler(oldAdapter,rand.nextLong()); |
| 174 | final BObserver bobs = new BObserver(varsToEncode,oldAdapter,sigmoidLoss,oldX); |
| 175 | final ThreadedReducer<BurstMap,BSampler,BObserver> reducer = new ThreadedReducer<BurstMap,BSampler,BObserver>(5,"BTable encoding"); |
| 176 | reducer.reduce(trainSource,bsampler,bobs); |
| 177 | log.info("done variable re-encoding scan"); |
| 178 | final BTable res = new BTable(); |
| 179 | res.sample = bsampler.sampler.data(); |
| 180 | log.info("start new adapter construction"); |
| 181 | // convert stats into encodings |
| 182 | res.oldAdapter = oldAdapter; |
| 183 | final Map<String,Map<String,VectorRow>> levelEncodings = new HashMap<String,Map<String,VectorRow>>(); // variable to level to vector |
| 184 | for(final Map.Entry<String,BStat> me: bobs.stats.entrySet()) { |
| 185 | final String variable = me.getKey(); |
| 186 | final BStat si = me.getValue(); |
| 187 | final ArrayList<GeneralIndicator> newCodes = encode(si,variable,oldAdapter,oldX); |
| 188 | if((null!=newCodes)&&(newCodes.size()>0)) { |
| 189 | final int edim = newCodes.size(); |
| 190 | final Map<String,VectorRow> enc = new HashMap<String,VectorRow>(); // level to vector |
| 191 | final String[] names = new String[edim]; |
| 192 | { |
| 193 | int giIndex = 0; |
| 194 | for(final GeneralIndicator gi: newCodes) { |
| 195 | names[giIndex] = gi.name; |
| 196 | ++giIndex; |
| 197 | } |
| 198 | } |
| 199 | for(final String level: si.levelStats.keySet()) { |
| 200 | final VectorRow row = new VectorRow(names,new double[edim],new int[edim]); |
| 201 | Arrays.fill(row.warmStartOutcome,-1); |
| 202 | enc.put(level,row); |
| 203 | int giIndex = 0; |
| 204 | for(final GeneralIndicator gi: newCodes) { |
| 205 | final Double lv = gi.levelEncodings.get(level); |
| 206 | row.levelEncodings[giIndex] = lv!=null?lv:0.0; |
| 207 | row.warmStartOutcome[giIndex] = gi.warmStartOutcome; // warm start assignment (not value) independent of level |
| 208 | ++giIndex; |
| 209 | } |
| 210 | } |
| 211 | levelEncodings.put(variable,enc); |
| 212 | } |
| 213 | } |
| 214 | // build new adapter |
| 215 | res.newAdapter = new VariableEncodings(oldAdapter.def(),oldAdapter.useIntercept(),oldAdapter.weightKey,levelEncodings); |
| 216 | // build warmstart vector |
| 217 | if(oldX!=null) { |
| 218 | final Map<String,VariableMapping> newAdaptions = new HashMap<String,VariableMapping>(); |