Method testBZ2Concatenation

test/org/apache/pig/test/TestBZip.java:580–669

Tests that Pig's Bzip2TextInputFormat throws an IOException when the input files to be loaded are actually a result of concatenating 2 or more bz2 files. It should not silently ignore part of the input data. When, hadoop's TextInpuFormat is used(PIG-3251), it should successfully read this concatena

()

     * successfully read this concatenated bzip file to the end.
     */
    @Test
    public void testBZ2Concatenation() throws Exception {
        String[] inputData1 = new String[] {
                "1\ta",
                "2\taa"
        };
        String[] inputData2 = new String[] {
                "1\tb",
                "2\tbb"
        };
        String[] inputDataMerged = new String[] {
                "1\ta",
                "2\taa",
                "1\tb",
                "2\tbb"
        };

        // bzip compressed input file1
        File in1 = folder.newFile("junit-in1.bz2");
        String compressedInputFileName1 = in1.getAbsolutePath();

        // file2
        File in2 = folder.newFile("junit-in2.bz2");
        String compressedInputFileName2 = in2.getAbsolutePath();

        String unCompressedInputFileName = "testRecordDelims-uncomp.txt";
        Util.createInputFile(cluster, unCompressedInputFileName, inputDataMerged);

        try {
            CBZip2OutputStream cos =
                new CBZip2OutputStream(new FileOutputStream(in1));
            for (int i = 0; i < inputData1.length; i++) {
                StringBuffer sb = new StringBuffer();
                sb.append(inputData1[i]).append("\n");
                byte bytes[] = sb.toString().getBytes();
                cos.write(bytes);
            }
            cos.close();

            CBZip2OutputStream cos2 =
                new CBZip2OutputStream(new FileOutputStream(in2));
            for (int i = 0; i < inputData2.length; i++) {
                StringBuffer sb = new StringBuffer();
                sb.append(inputData2[i]).append("\n");
                byte bytes[] = sb.toString().getBytes();
                cos2.write(bytes);
            }
            cos2.close();

            // cat
            catInto(compressedInputFileName2, compressedInputFileName1);
            Util.copyFromLocalToCluster(cluster, compressedInputFileName1,
                    compressedInputFileName1);

            // pig script to read uncompressed input
            String script = "a = load '" + Util.encodeEscape(unCompressedInputFileName) +"';";
            PigServer pig = new PigServer(cluster.getExecType(), properties);
            pig.registerQuery(script);