| 151 | } |
| 152 | |
| 153 | static void main(Configuration conf, String[] args) throws ParseException { |
| 154 | CommandLine cli = parseCommandLine(args); |
| 155 | if (cli.hasOption('h') || cli.getArgs().length == 0) { |
| 156 | new HelpFormatter().printHelp("java -jar orc-tools-*.jar scan", |
| 157 | OPTIONS); |
| 158 | System.exit(1); |
| 159 | } else { |
| 160 | final boolean printSchema = cli.hasOption('s'); |
| 161 | final boolean printExceptions = cli.hasOption('v'); |
| 162 | List<String> badFiles = new ArrayList<>(); |
| 163 | for (String file : cli.getArgs()) { |
| 164 | try (Reader reader = FileDump.getReader(new Path(file), conf, badFiles)) { |
| 165 | if (reader != null) { |
| 166 | TypeDescription schema = reader.getSchema(); |
| 167 | if (printSchema) { |
| 168 | System.out.println(schema.toJson()); |
| 169 | } |
| 170 | VectorizedRowBatch batch = schema.createRowBatch( |
| 171 | TypeDescription.RowBatchVersion.USE_DECIMAL64, |
| 172 | calculateBestVectorSize(reader.getRowIndexStride())); |
| 173 | final int batchSize = batch.getMaxSize(); |
| 174 | long badBatches = 0; |
| 175 | long currentRow = 0; |
| 176 | long goodRows = 0; |
| 177 | try (RecordReader rows = reader.rows()) { |
| 178 | while (currentRow < reader.getNumberOfRows()) { |
| 179 | currentRow = rows.getRowNumber(); |
| 180 | try { |
| 181 | if (!rows.nextBatch(batch)) { |
| 182 | break; |
| 183 | } |
| 184 | goodRows += batch.size; |
| 185 | } catch (Exception e) { |
| 186 | badBatches += 1; |
| 187 | LocationInfo current = findStripeInfo(reader, currentRow); |
| 188 | LocationInfo recover = findRecoveryPoint(reader, current, batchSize); |
| 189 | System.out.println("Unable to read batch at " + current + |
| 190 | ", recovery at " + recover); |
| 191 | if (printExceptions) { |
| 192 | e.printStackTrace(); |
| 193 | } |
| 194 | findBadColumns(reader, current, batchSize, reader.getSchema(), |
| 195 | new boolean[reader.getSchema().getMaximumId() + 1]); |
| 196 | // If we are at the end of the file, get out |
| 197 | if (recover.row >= reader.getNumberOfRows()) { |
| 198 | break; |
| 199 | } else { |
| 200 | rows.seekToRow(recover.row); |
| 201 | } |
| 202 | } |
| 203 | } |
| 204 | } |
| 205 | if (badBatches != 0) { |
| 206 | badFiles.add(file); |
| 207 | } |
| 208 | System.out.printf("File: %s, bad batches: %d, rows: %d/%d%n", file, |
| 209 | badBatches, goodRows, reader.getNumberOfRows()); |
| 210 | } |