Copies the files from remote to local filesystem. When 'multipleFiles' is set the path could point to multiple files through globs or a directory. In this case, return array contains multiple files, otherwise a single file is returned. If pig.jars.relative.to.dfs is true then a relative path is ass
(Properties properties,
String filePath,
boolean multipleFiles)
| 783 | * @return |
| 784 | */ |
| 785 | private static FetchFileRet[] fetchFilesInternal(Properties properties, |
| 786 | String filePath, |
| 787 | boolean multipleFiles) throws IOException { |
| 788 | |
| 789 | Path path = new Path(filePath); |
| 790 | if (path.getName().isEmpty()) { |
| 791 | return new FetchFileRet[0]; |
| 792 | } |
| 793 | URI uri = path.toUri(); |
| 794 | Configuration conf = new Configuration(); |
| 795 | ConfigurationUtil.mergeConf(conf, ConfigurationUtil.toConfiguration(properties)); |
| 796 | |
| 797 | // if there is no schema or if the schema is "local", then it is |
| 798 | // expected to be a local path. |
| 799 | |
| 800 | FileSystem localFs = FileSystem.getLocal(conf); |
| 801 | FileSystem srcFs; |
| 802 | if ( (!"true".equals(properties.getProperty("pig.jars.relative.to.dfs")) |
| 803 | && uri.getScheme() == null )|| |
| 804 | // For Windows local files |
| 805 | (uri.getScheme() == null && uri.getPath().matches("^/[A-Za-z]:.*")) || |
| 806 | (uri.getScheme() != null && uri.getScheme().equals("local")) |
| 807 | ) { |
| 808 | srcFs = localFs; |
| 809 | } else { |
| 810 | srcFs = path.getFileSystem(conf); |
| 811 | } |
| 812 | |
| 813 | FileStatus[] files; |
| 814 | |
| 815 | if (multipleFiles) { |
| 816 | files = srcFs.globStatus(path); |
| 817 | } else { |
| 818 | files = new FileStatus[]{ srcFs.getFileStatus(path) }; |
| 819 | } |
| 820 | if (files == null || files.length == 0) { |
| 821 | throw new ExecException("file '" + filePath + "' does not exist.", 101, PigException.INPUT); |
| 822 | } |
| 823 | |
| 824 | FetchFileRet[] fetchFiles = new FetchFileRet[files.length]; |
| 825 | int idx = 0; |
| 826 | |
| 827 | for(FileStatus file : files) { |
| 828 | // should throw an exception if this is not a file? |
| 829 | |
| 830 | String pathname = file.getPath().toUri().getPath(); |
| 831 | String filename = file.getPath().getName(); |
| 832 | |
| 833 | if (srcFs == localFs) { |
| 834 | fetchFiles[idx++] = new FetchFileRet(new File(pathname), false); |
| 835 | } else { |
| 836 | // fetch from remote: |
| 837 | File dest = new File(localTempDir, filename); |
| 838 | dest.deleteOnExit(); |
| 839 | try { |
| 840 | srcFs.copyToLocalFile(file.getPath(), new Path(dest.getAbsolutePath())); |
| 841 | } catch (IOException e) { |
| 842 | throw new ExecException("Could not copy " + filePath + " to local destination " + dest, 101, PigException.INPUT, e); |
no test coverage detected