This method returns an ORC reader object if the specified file is readable. If the specified file has side file (_flush_length) file, then max footer offset will be read from the side file and orc reader will be created from that offset. Since both data file and side file use hflush() for flushing t
(final Path path, final Configuration conf,
final List<String> corruptFiles)
| 176 | * @throws IOException |
| 177 | */ |
| 178 | static Reader getReader(final Path path, final Configuration conf, |
| 179 | final List<String> corruptFiles) throws IOException { |
| 180 | FileSystem fs = path.getFileSystem(conf); |
| 181 | long dataFileLen = fs.getFileStatus(path).getLen(); |
| 182 | System.err.println("Processing data file " + path + " [length: " + dataFileLen + "]"); |
| 183 | Path sideFile = OrcAcidUtils.getSideFile(path); |
| 184 | final boolean sideFileExists = fs.exists(sideFile); |
| 185 | boolean openDataFile = false; |
| 186 | boolean openSideFile = false; |
| 187 | if (fs instanceof DistributedFileSystem dfs) { |
| 188 | openDataFile = !dfs.isFileClosed(path); |
| 189 | openSideFile = sideFileExists && !dfs.isFileClosed(sideFile); |
| 190 | } |
| 191 | |
| 192 | if (openDataFile || openSideFile) { |
| 193 | if (openDataFile && openSideFile) { |
| 194 | System.err.println("Unable to perform file dump as " + path + " and " + sideFile + |
| 195 | " are still open for writes."); |
| 196 | } else if (openSideFile) { |
| 197 | System.err.println("Unable to perform file dump as " + sideFile + |
| 198 | " is still open for writes."); |
| 199 | } else { |
| 200 | System.err.println("Unable to perform file dump as " + path + |
| 201 | " is still open for writes."); |
| 202 | } |
| 203 | |
| 204 | return null; |
| 205 | } |
| 206 | |
| 207 | Reader reader = null; |
| 208 | if (sideFileExists) { |
| 209 | final long maxLen = OrcAcidUtils.getLastFlushLength(fs, path); |
| 210 | final long sideFileLen = fs.getFileStatus(sideFile).getLen(); |
| 211 | System.err.println("Found flush length file " + sideFile |
| 212 | + " [length: " + sideFileLen + ", maxFooterOffset: " + maxLen + "]"); |
| 213 | // no offsets read from side file |
| 214 | if (maxLen == -1) { |
| 215 | |
| 216 | // if data file is larger than last flush length, then additional data could be recovered |
| 217 | if (dataFileLen > maxLen) { |
| 218 | System.err.println("Data file has more data than max footer offset:" + maxLen + |
| 219 | ". Adding data file to recovery list."); |
| 220 | if (corruptFiles != null) { |
| 221 | corruptFiles.add(path.toUri().toString()); |
| 222 | } |
| 223 | } |
| 224 | return null; |
| 225 | } |
| 226 | |
| 227 | try { |
| 228 | reader = OrcFile.createReader(path, OrcFile.readerOptions(conf).maxLength(maxLen)); |
| 229 | |
| 230 | // if data file is larger than last flush length, then additional data could be recovered |
| 231 | if (dataFileLen > maxLen) { |
| 232 | System.err.println("Data file has more data than max footer offset:" + maxLen + |
| 233 | ". Adding data file to recovery list."); |
| 234 | if (corruptFiles != null) { |
| 235 | corruptFiles.add(path.toUri().toString()); |
no test coverage detected