Class TestLargeFile

test/org/apache/pig/test/TestLargeFile.java:44–173

This class tests pig behavior with large file spanning multiple blocks along with group and count functions Order and Distinct functions are also tested. This test takes long time because of the large test files.

 * Order and Distinct functions are also tested. This test takes long time because of the large test files.
 */
public class TestLargeFile {

    File datFile;

    private long defaultBlockSize = (new Configuration()).getLong("dfs.block.size", 0);

    private long total = defaultBlockSize >> 1;
    private int max_rand = 500;
    static MiniCluster cluster = MiniCluster.buildCluster();

    Integer[] COUNT = new Integer[max_rand];

    PigServer pig;
    String fileName, tmpFile1;

    @Before
    public void setUp() throws Exception{

        System.out.println("Generating test data...");
        System.out.println("Default block size = " + defaultBlockSize);
        System.out.println("Total no. of iterations to run for the test data = " + total);

        datFile = File.createTempFile("StoreTest", ".dat");

        FileOutputStream dat = new FileOutputStream(datFile);

        Random rand = new Random();

        for(int i = 0; i < max_rand; i++) {
            COUNT[i] = 0;
        }

        for(long i = 0; i < total; i++) {
            Integer x = new Integer(rand.nextInt(max_rand));
            COUNT[x.intValue()]++;
            dat.write((x.toString() + "\n").getBytes());
        }

        dat.close();

        pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());

        fileName = "'" + FileLocalizer.hadoopify(datFile.toString(), pig.getPigContext()) + "'";
        tmpFile1 = "'" + FileLocalizer.getTemporaryPath(pig.getPigContext()).toString() + "'";

        datFile.delete();
    }

    @After
    public void tearDown() throws Exception {
        pig.shutdown();
    }

    @AfterClass
    public static void oneTimeTearDown() throws Exception {
        cluster.shutDown();
    }