()
| 63 | } |
| 64 | |
| 65 | @Override |
| 66 | protected void setupMiniDfsAndMrClusters() { |
| 67 | try { |
| 68 | final int dataNodes = 4; // There will be 4 data nodes |
| 69 | final int taskTrackers = 4; // There will be 4 task tracker nodes |
| 70 | |
| 71 | System.setProperty("hadoop.log.dir", "build/test/logs"); |
| 72 | // Create the dir that holds hadoop-site.xml file |
| 73 | // Delete if hadoop-site.xml exists already |
| 74 | CONF_DIR.mkdirs(); |
| 75 | if(CONF_FILE.exists()) { |
| 76 | CONF_FILE.delete(); |
| 77 | } |
| 78 | |
| 79 | // Builds and starts the mini dfs and mapreduce clusters |
| 80 | Configuration config = new Configuration(); |
| 81 | config.set("yarn.scheduler.capacity.root.queues", "default"); |
| 82 | config.set("yarn.scheduler.capacity.root.default.capacity", "100"); |
| 83 | config.set("yarn.scheduler.capacity.maximum-am-resource-percent", "0.1"); |
| 84 | m_dfs = new MiniDFSCluster(config, dataNodes, true, null); |
| 85 | m_fileSys = m_dfs.getFileSystem(); |
| 86 | m_dfs_conf = m_dfs.getConfiguration(0); |
| 87 | |
| 88 | //Create user home directory |
| 89 | m_fileSys.mkdirs(m_fileSys.getWorkingDirectory()); |
| 90 | |
| 91 | m_mr = new MiniMRYarnCluster("PigMiniCluster", taskTrackers); |
| 92 | m_mr.init(m_dfs_conf); |
| 93 | m_mr.start(); |
| 94 | |
| 95 | // Write the necessary config info to hadoop-site.xml |
| 96 | m_mr_conf = new Configuration(m_mr.getConfig()); |
| 97 | |
| 98 | m_conf = m_mr_conf; |
| 99 | m_conf.set(FileSystem.FS_DEFAULT_NAME_KEY, m_dfs_conf.get(FileSystem.FS_DEFAULT_NAME_KEY)); |
| 100 | m_conf.unset(MRConfiguration.JOB_CACHE_FILES); |
| 101 | |
| 102 | m_conf.setInt(MRConfiguration.IO_SORT_MB, 50); |
| 103 | m_conf.set(MRConfiguration.CHILD_JAVA_OPTS, "-Xmx384m"); |
| 104 | m_conf.setInt(MRJobConfig.MAP_MEMORY_MB, 512); |
| 105 | m_conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 512); |
| 106 | m_conf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx384m"); |
| 107 | m_conf.setInt(MRJobConfig.MR_AM_VMEM_MB, 512); |
| 108 | |
| 109 | m_conf.setInt(MRConfiguration.SUMIT_REPLICATION, 2); |
| 110 | m_conf.setInt(MRConfiguration.MAP_MAX_ATTEMPTS, 2); |
| 111 | m_conf.setInt(MRConfiguration.REDUCE_MAX_ATTEMPTS, 2); |
| 112 | m_conf.set("dfs.datanode.address", "0.0.0.0:0"); |
| 113 | m_conf.set("dfs.datanode.http.address", "0.0.0.0:0"); |
| 114 | m_conf.set("pig.jobcontrol.sleep", "100"); |
| 115 | m_conf.writeXml(new FileOutputStream(CONF_FILE)); |
| 116 | m_fileSys.copyFromLocalFile(new Path(CONF_FILE.getAbsoluteFile().toString()), |
| 117 | new Path("/pigtest/conf/hadoop-site.xml")); |
| 118 | DistributedCache.addFileToClassPath(new Path("/pigtest/conf/hadoop-site.xml"), m_conf); |
| 119 | |
| 120 | System.err.println("XXX: Setting " + FileSystem.FS_DEFAULT_NAME_KEY + " to: " + m_conf.get(FileSystem.FS_DEFAULT_NAME_KEY)); |
| 121 | // Set the system properties needed by Pig |
| 122 | System.setProperty("cluster", m_conf.get(MRConfiguration.JOB_TRACKER)); |
nothing calls this directly
no test coverage detected