This test checks records that align perfectly on bzip block boundaries and hdfs block boundaries
()
| 115 | * bzip block boundaries and hdfs block boundaries |
| 116 | */ |
| 117 | @Test |
| 118 | public void testBZip2Aligned() throws Throwable { |
| 119 | int offsets[] = { 219642, 219643, 219644, 552019, 552020 }; |
| 120 | for(int i = 1; i < offsets.length; i ++) { |
| 121 | |
| 122 | Properties props = new Properties(); |
| 123 | for (Entry<Object, Object> entry : cluster.getProperties().entrySet()) { |
| 124 | props.put(entry.getKey(), entry.getValue()); |
| 125 | } |
| 126 | props.setProperty(MRConfiguration.MAX_SPLIT_SIZE, Integer.toString(offsets[i])); |
| 127 | PigContext pigContext = new PigContext(cluster.getExecType(), props); |
| 128 | PigServer pig = new PigServer(pigContext); |
| 129 | pig.registerQuery("a = load '" |
| 130 | + Util.generateURI( |
| 131 | "file:test/org/apache/pig/test/data/bzipTest.bz2", |
| 132 | pig.getPigContext()) + "';"); |
| 133 | //pig.registerQuery("a = foreach (group (load 'file:test/org/apache/pig/test/data/bzipTest.bz2') all) generate COUNT($1);"); |
| 134 | Iterator<Tuple> it = pig.openIterator("a"); |
| 135 | int count = 0; |
| 136 | while(it.hasNext()) { |
| 137 | Tuple t = it.next(); |
| 138 | String s = t.get(0).toString(); |
| 139 | s = s.substring(0, 7); |
| 140 | assertEquals("Using blocksize " + offsets[i] + " problem with " + t, count, Integer.parseInt(s, 16)); |
| 141 | count++; |
| 142 | } |
| 143 | //assertEquals("1000000", it.next().getField(0)); |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | public Double bigGroupAll( File tmpFile ) throws Throwable { |
| 148 |
nothing calls this directly
no test coverage detected