You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2014/03/26 19:14:39 UTC
svn commit: r1581977 [4/5] - in /hive/trunk: ./
common/src/java/org/apache/hadoop/hive/common/
itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/
metastore/src/java/org/apache/hadoop/hive/metastore/
metastore/src/java/org/apache/hadoop/hi...
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java?rev=1581977&r1=1581976&r2=1581977&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java Wed Mar 26 18:14:37 2014
@@ -23,14 +23,20 @@ import static org.junit.Assert.assertTru
import java.io.DataInput;
import java.io.DataOutput;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
+import java.util.Set;
+import java.util.TreeSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
@@ -42,21 +48,34 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
+import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
import org.apache.hadoop.hive.ql.io.FSRecordWriter;
+import org.apache.hadoop.hive.ql.io.HiveInputFormat;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.InputFormatChecker;
+import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.shims.CombineHiveKey;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
@@ -154,13 +173,14 @@ public class TestInputOutputFormat {
public void testFileGenerator() throws Exception {
TestContext context = new TestContext(conf);
MockFileSystem fs = new MockFileSystem(conf,
- new MockFile("/a/b/part-00", 1000, new byte[0]),
- new MockFile("/a/b/part-01", 1000, new byte[0]),
- new MockFile("/a/b/_part-02", 1000, new byte[0]),
- new MockFile("/a/b/.part-03", 1000, new byte[0]),
- new MockFile("/a/b/part-04", 1000, new byte[0]));
+ new MockFile("mock:/a/b/part-00", 1000, new byte[0]),
+ new MockFile("mock:/a/b/part-01", 1000, new byte[0]),
+ new MockFile("mock:/a/b/_part-02", 1000, new byte[0]),
+ new MockFile("mock:/a/b/.part-03", 1000, new byte[0]),
+ new MockFile("mock:/a/b/part-04", 1000, new byte[0]));
OrcInputFormat.FileGenerator gen =
- new OrcInputFormat.FileGenerator(context, fs, new Path("/a/b"));
+ new OrcInputFormat.FileGenerator(context, fs,
+ new MockPath(fs, "mock:/a/b"));
gen.run();
if (context.getErrors().size() > 0) {
for(Throwable th: context.getErrors()) {
@@ -170,34 +190,50 @@ public class TestInputOutputFormat {
}
assertEquals(-1, context.getSchedulers());
assertEquals(3, context.queue.size());
- assertEquals(new Path("/a/b/part-00"),
+ assertEquals(new Path("mock:/a/b/part-00"),
((OrcInputFormat.SplitGenerator) context.queue.get(0)).getPath());
- assertEquals(new Path("/a/b/part-01"),
+ assertEquals(new Path("mock:/a/b/part-01"),
((OrcInputFormat.SplitGenerator) context.queue.get(1)).getPath());
- assertEquals(new Path("/a/b/part-04"),
+ assertEquals(new Path("mock:/a/b/part-04"),
((OrcInputFormat.SplitGenerator) context.queue.get(2)).getPath());
}
- static final Charset UTF8 = Charset.forName("UTF-8");
-
- static class MockBlock {
+ public static class MockBlock {
int offset;
int length;
final String[] hosts;
- MockBlock(String... hosts) {
+ public MockBlock(String... hosts) {
this.hosts = hosts;
}
+
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append("block{offset: ");
+ buffer.append(offset);
+ buffer.append(", length: ");
+ buffer.append(length);
+ buffer.append(", hosts: [");
+ for(int i=0; i < hosts.length; i++) {
+ if (i != 0) {
+ buffer.append(", ");
+ }
+ buffer.append(hosts[i]);
+ }
+ buffer.append("]}");
+ return buffer.toString();
+ }
}
- static class MockFile {
+ public static class MockFile {
final Path path;
- final int blockSize;
- final int length;
- final MockBlock[] blocks;
- final byte[] content;
+ int blockSize;
+ int length;
+ MockBlock[] blocks;
+ byte[] content;
- MockFile(String path, int blockSize, byte[] content, MockBlock... blocks) {
+ public MockFile(String path, int blockSize, byte[] content,
+ MockBlock... blocks) {
this.path = new Path(path);
this.blockSize = blockSize;
this.blocks = blocks;
@@ -210,6 +246,25 @@ public class TestInputOutputFormat {
offset += block.length;
}
}
+
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append("mockFile{path: ");
+ buffer.append(path.toString());
+ buffer.append(", blkSize: ");
+ buffer.append(blockSize);
+ buffer.append(", len: ");
+ buffer.append(length);
+ buffer.append(", blocks: [");
+ for(int i=0; i < blocks.length; i++) {
+ if (i != 0) {
+ buffer.append(", ");
+ }
+ buffer.append(blocks[i]);
+ }
+ buffer.append("]}");
+ return buffer.toString();
+ }
}
static class MockInputStream extends FSInputStream {
@@ -244,13 +299,66 @@ public class TestInputOutputFormat {
}
}
- static class MockFileSystem extends FileSystem {
- final MockFile[] files;
+ public static class MockPath extends Path {
+ private final FileSystem fs;
+ public MockPath(FileSystem fs, String path) {
+ super(path);
+ this.fs = fs;
+ }
+ @Override
+ public FileSystem getFileSystem(Configuration conf) {
+ return fs;
+ }
+ }
+
+ public static class MockOutputStream extends FSDataOutputStream {
+ private final MockFile file;
+
+ public MockOutputStream(MockFile file) throws IOException {
+ super(new DataOutputBuffer(), null);
+ this.file = file;
+ }
+
+ public void setBlocks(MockBlock... blocks) {
+ file.blocks = blocks;
+ int offset = 0;
+ int i = 0;
+ while (offset < file.length && i < blocks.length) {
+ blocks[i].offset = offset;
+ blocks[i].length = Math.min(file.length - offset, file.blockSize);
+ offset += blocks[i].length;
+ i += 1;
+ }
+ }
+
+ public void close() throws IOException {
+ super.close();
+ DataOutputBuffer buf = (DataOutputBuffer) getWrappedStream();
+ file.length = buf.getLength();
+ file.content = new byte[file.length];
+ System.arraycopy(buf.getData(), 0, file.content, 0, file.length);
+ }
+ }
+
+ public static class MockFileSystem extends FileSystem {
+ final List<MockFile> files = new ArrayList<MockFile>();
Path workingDir = new Path("/");
- MockFileSystem(Configuration conf, MockFile... files) {
+ public MockFileSystem() {
+ // empty
+ }
+
+ public void initialize(URI uri, Configuration conf) {
setConf(conf);
- this.files = files;
+ }
+
+ public MockFileSystem(Configuration conf, MockFile... files) {
+ setConf(conf);
+ this.files.addAll(Arrays.asList(files));
+ }
+
+ void clear() {
+ files.clear();
}
@Override
@@ -274,17 +382,30 @@ public class TestInputOutputFormat {
@Override
public FSDataOutputStream create(Path path, FsPermission fsPermission,
- boolean b, int i, short i2, long l,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
Progressable progressable
) throws IOException {
- throw new UnsupportedOperationException("no writes");
+ MockFile file = null;
+ for(MockFile currentFile: files) {
+ if (currentFile.path.equals(path)) {
+ file = currentFile;
+ break;
+ }
+ }
+ if (file == null) {
+ file = new MockFile(path.toString(), (int) blockSize, new byte[0]);
+ files.add(file);
+ }
+ return new MockOutputStream(file);
}
@Override
- public FSDataOutputStream append(Path path, int i,
+ public FSDataOutputStream append(Path path, int bufferSize,
Progressable progressable
) throws IOException {
- throw new UnsupportedOperationException("no writes");
+ return create(path, FsPermission.getDefault(), true, bufferSize,
+ (short) 3, 256 * 1024, progressable);
}
@Override
@@ -304,12 +425,29 @@ public class TestInputOutputFormat {
@Override
public FileStatus[] listStatus(Path path) throws IOException {
+ path = path.makeQualified(this);
List<FileStatus> result = new ArrayList<FileStatus>();
+ String pathname = path.toString();
+ String pathnameAsDir = pathname + "/";
+ Set<String> dirs = new TreeSet<String>();
for(MockFile file: files) {
- if (file.path.getParent().equals(path)) {
- result.add(getFileStatus(file.path));
+ String filename = file.path.toString();
+ if (pathname.equals(filename)) {
+ return new FileStatus[]{createStatus(file)};
+ } else if (filename.startsWith(pathnameAsDir)) {
+ String tail = filename.substring(pathnameAsDir.length());
+ int nextSlash = tail.indexOf('/');
+ if (nextSlash > 0) {
+ dirs.add(tail.substring(0, nextSlash));
+ } else {
+ result.add(createStatus(file));
+ }
}
}
+ // for each directory add it once
+ for(String dir: dirs) {
+ result.add(createDirectory(new MockPath(this, pathnameAsDir + dir)));
+ }
return result.toArray(new FileStatus[result.size()]);
}
@@ -328,16 +466,29 @@ public class TestInputOutputFormat {
return false;
}
+ private FileStatus createStatus(MockFile file) {
+ return new FileStatus(file.length, false, 1, file.blockSize, 0, 0,
+ FsPermission.createImmutable((short) 644), "owen", "group",
+ file.path);
+ }
+
+ private FileStatus createDirectory(Path dir) {
+ return new FileStatus(0, true, 0, 0, 0, 0,
+ FsPermission.createImmutable((short) 755), "owen", "group", dir);
+ }
+
@Override
public FileStatus getFileStatus(Path path) throws IOException {
+ path = path.makeQualified(this);
+ String pathnameAsDir = path.toString() + "/";
for(MockFile file: files) {
if (file.path.equals(path)) {
- return new FileStatus(file.length, false, 1, file.blockSize, 0, 0,
- FsPermission.createImmutable((short) 644), "owen", "group",
- file.path);
+ return createStatus(file);
+ } else if (file.path.toString().startsWith(pathnameAsDir)) {
+ return createDirectory(path);
}
}
- return null;
+ throw new FileNotFoundException("File " + path + " does not exist");
}
@Override
@@ -358,6 +509,20 @@ public class TestInputOutputFormat {
}
return new BlockLocation[0];
}
+
+ @Override
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append("mockFs{files:[");
+ for(int i=0; i < files.size(); ++i) {
+ if (i != 0) {
+ buffer.append(", ");
+ }
+ buffer.append(files.get(i));
+ }
+ buffer.append("]}");
+ return buffer.toString();
+ }
}
static void fill(DataOutputBuffer out, long length) throws IOException {
@@ -416,7 +581,7 @@ public class TestInputOutputFormat {
public void testAddSplit() throws Exception {
// create a file with 5 blocks spread around the cluster
MockFileSystem fs = new MockFileSystem(conf,
- new MockFile("/a/file", 500,
+ new MockFile("mock:/a/file", 500,
createMockOrcFile(197, 300, 600, 200, 200, 100, 100, 100, 100, 100),
new MockBlock("host1-1", "host1-2", "host1-3"),
new MockBlock("host2-1", "host0", "host2-3"),
@@ -426,12 +591,13 @@ public class TestInputOutputFormat {
OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
OrcInputFormat.SplitGenerator splitter =
new OrcInputFormat.SplitGenerator(context, fs,
- fs.getFileStatus(new Path("/a/file")), null);
+ fs.getFileStatus(new Path("/a/file")), null, true,
+ new ArrayList<Long>(), true);
splitter.createSplit(0, 200, null);
- OrcInputFormat.Context.FileSplitInfo result = context.getResult(-1);
+ OrcSplit result = context.getResult(-1);
assertEquals(0, result.getStart());
assertEquals(200, result.getLength());
- assertEquals("/a/file", result.getPath().toString());
+ assertEquals("mock:/a/file", result.getPath().toString());
String[] locs = result.getLocations();
assertEquals(3, locs.length);
assertEquals("host1-1", locs[0]);
@@ -457,7 +623,7 @@ public class TestInputOutputFormat {
long[] stripeSizes =
new long[]{197, 300, 600, 200, 200, 100, 100, 100, 100, 100};
MockFileSystem fs = new MockFileSystem(conf,
- new MockFile("/a/file", 500,
+ new MockFile("mock:/a/file", 500,
createMockOrcFile(stripeSizes),
new MockBlock("host1-1", "host1-2", "host1-3"),
new MockBlock("host2-1", "host0", "host2-3"),
@@ -469,7 +635,8 @@ public class TestInputOutputFormat {
OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
OrcInputFormat.SplitGenerator splitter =
new OrcInputFormat.SplitGenerator(context, fs,
- fs.getFileStatus(new Path("/a/file")), null);
+ fs.getFileStatus(new Path("/a/file")), null, true,
+ new ArrayList<Long>(), true);
splitter.run();
if (context.getErrors().size() > 0) {
for(Throwable th: context.getErrors()) {
@@ -477,7 +644,7 @@ public class TestInputOutputFormat {
}
throw new IOException("Errors during splitting");
}
- OrcInputFormat.Context.FileSplitInfo result = context.getResult(0);
+ OrcSplit result = context.getResult(0);
assertEquals(3, result.getStart());
assertEquals(497, result.getLength());
result = context.getResult(1);
@@ -497,7 +664,8 @@ public class TestInputOutputFormat {
conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 0);
context = new OrcInputFormat.Context(conf);
splitter = new OrcInputFormat.SplitGenerator(context, fs,
- fs.getFileStatus(new Path("/a/file")), null);
+ fs.getFileStatus(new Path("/a/file")), null, true, new ArrayList<Long>(),
+ true);
splitter.run();
if (context.getErrors().size() > 0) {
for(Throwable th: context.getErrors()) {
@@ -512,6 +680,7 @@ public class TestInputOutputFormat {
}
@Test
+ @SuppressWarnings("unchecked,deprecation")
public void testInOutFormat() throws Exception {
Properties properties = new Properties();
StructObjectInspector inspector;
@@ -625,8 +794,8 @@ public class TestInputOutputFormat {
}
@Test
+ @SuppressWarnings("unchecked,deprecation")
public void testMROutput() throws Exception {
- JobConf job = new JobConf(conf);
Properties properties = new Properties();
StructObjectInspector inspector;
synchronized (TestOrcFile.class) {
@@ -682,8 +851,8 @@ public class TestInputOutputFormat {
}
@Test
+ @SuppressWarnings("deprecation")
public void testEmptyFile() throws Exception {
- JobConf job = new JobConf(conf);
Properties properties = new Properties();
HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat();
FSRecordWriter writer =
@@ -720,8 +889,8 @@ public class TestInputOutputFormat {
}
@Test
+ @SuppressWarnings("unchecked,deprecation")
public void testDefaultTypes() throws Exception {
- JobConf job = new JobConf(conf);
Properties properties = new Properties();
StructObjectInspector inspector;
synchronized (TestOrcFile.class) {
@@ -780,4 +949,293 @@ public class TestInputOutputFormat {
assertEquals(false, reader.next(key, value));
reader.close();
}
+
+ /**
+ * Create a mock execution environment that has enough detail that
+ * ORC, vectorization, HiveInputFormat, and CombineHiveInputFormat don't
+ * explode.
+ * @param workDir a local filesystem work directory
+ * @param warehouseDir a mock filesystem warehouse directory
+ * @param objectInspector object inspector for the row
+ * @param isVectorized should run vectorized
+ * @return a JobConf that contains the necessary information
+ * @throws IOException
+ */
+ JobConf createMockExecutionEnvironment(Path workDir,
+ Path warehouseDir,
+ ObjectInspector objectInspector,
+ boolean isVectorized
+ ) throws IOException {
+ JobConf conf = new JobConf();
+ conf.set("hive.exec.plan", workDir.toString());
+ conf.set("mapred.job.tracker", "local");
+ conf.set("hive.vectorized.execution.enabled", Boolean.toString(isVectorized));
+ conf.set("fs.mock.impl", MockFileSystem.class.getName());
+ conf.set("mapred.mapper.class", ExecMapper.class.getName());
+ Path root = new Path(warehouseDir, "table/p=0");
+ conf.set("mapred.input.dir", root.toString());
+ StringBuilder columnIds = new StringBuilder();
+ StringBuilder columnNames = new StringBuilder();
+ StringBuilder columnTypes = new StringBuilder();
+ StructObjectInspector structOI = (StructObjectInspector) objectInspector;
+ List<? extends StructField> fields = structOI.getAllStructFieldRefs();
+ int numCols = fields.size();
+ for(int i=0; i < numCols; ++i) {
+ if (i != 0) {
+ columnIds.append(',');
+ columnNames.append(',');
+ columnTypes.append(',');
+ }
+ columnIds.append(i);
+ columnNames.append(fields.get(i).getFieldName());
+ columnTypes.append(fields.get(i).getFieldObjectInspector().getTypeName());
+ }
+ conf.set("hive.io.file.readcolumn.ids", columnIds.toString());
+ conf.set("partition_columns", "p");
+ MockFileSystem fs = (MockFileSystem) warehouseDir.getFileSystem(conf);
+ fs.clear();
+
+ Properties tblProps = new Properties();
+ tblProps.put("serialization.lib", OrcSerde.class.getName());
+ tblProps.put("columns", columnNames.toString());
+ tblProps.put("columns.types", columnTypes.toString());
+ TableDesc tbl = new TableDesc(OrcInputFormat.class, OrcOutputFormat.class,
+ tblProps);
+ LinkedHashMap<String, String> partSpec =
+ new LinkedHashMap<String, String>();
+ PartitionDesc part = new PartitionDesc(tbl, partSpec);
+
+ MapWork mapWork = new MapWork();
+ mapWork.setVectorMode(isVectorized);
+ mapWork.setUseBucketizedHiveInputFormat(false);
+ LinkedHashMap<String, ArrayList<String>> aliasMap =
+ new LinkedHashMap<String, ArrayList<String>>();
+ ArrayList<String> aliases = new ArrayList<String>();
+ aliases.add("tbl");
+ aliasMap.put(root.toString(), aliases);
+ mapWork.setPathToAliases(aliasMap);
+ LinkedHashMap<String, PartitionDesc> partMap =
+ new LinkedHashMap<String, PartitionDesc>();
+ partMap.put(root.toString(), part);
+ mapWork.setPathToPartitionInfo(partMap);
+ mapWork.setScratchColumnMap(new HashMap<String, Map<String, Integer>>());
+ mapWork.setScratchColumnVectorTypes(new HashMap<String,
+ Map<Integer, String>>());
+
+ // write the plan out
+ FileSystem localFs = FileSystem.getLocal(conf).getRaw();
+ FSDataOutputStream planStream =
+ localFs.create(new Path(workDir, "map.xml"));
+ Utilities.serializePlan(mapWork, planStream, conf);
+ planStream.close();
+ return conf;
+ }
+
+ /**
+ * Test vectorization, non-acid, non-combine.
+ * @throws Exception
+ */
+ @Test
+ public void testVectorization() throws Exception {
+ // get the object inspector for MyRow
+ StructObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = (StructObjectInspector)
+ ObjectInspectorFactory.getReflectionObjectInspector(MyRow.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ JobConf conf = createMockExecutionEnvironment(workDir, new Path("mock:///"),
+ inspector, true);
+
+ // write the orc file to the mock file system
+ Writer writer =
+ OrcFile.createWriter(new Path(conf.get("mapred.input.dir") + "/0_0"),
+ OrcFile.writerOptions(conf).blockPadding(false)
+ .bufferSize(1024).inspector(inspector));
+ for(int i=0; i < 10; ++i) {
+ writer.addRow(new MyRow(i, 2*i));
+ }
+ writer.close();
+ ((MockOutputStream) ((WriterImpl) writer).getStream())
+ .setBlocks(new MockBlock("host0", "host1"));
+
+ // call getsplits
+ HiveInputFormat<?,?> inputFormat =
+ new HiveInputFormat<WritableComparable, Writable>();
+ InputSplit[] splits = inputFormat.getSplits(conf, 10);
+ assertEquals(1, splits.length);
+
+ org.apache.hadoop.mapred.RecordReader<NullWritable, VectorizedRowBatch>
+ reader = inputFormat.getRecordReader(splits[0], conf, Reporter.NULL);
+ NullWritable key = reader.createKey();
+ VectorizedRowBatch value = reader.createValue();
+ assertEquals(true, reader.next(key, value));
+ assertEquals(10, value.count());
+ LongColumnVector col0 = (LongColumnVector) value.cols[0];
+ for(int i=0; i < 10; i++) {
+ assertEquals("checking " + i, i, col0.vector[i]);
+ }
+ assertEquals(false, reader.next(key, value));
+ }
+
+ // test acid with vectorization, no combine
+ @Test
+ public void testVectorizationWithAcid() throws Exception {
+ // get the object inspector for MyRow
+ StructObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = (StructObjectInspector)
+ ObjectInspectorFactory.getReflectionObjectInspector(MyRow.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ JobConf conf = createMockExecutionEnvironment(workDir, new Path("mock:///"),
+ inspector, true);
+
+ // write the orc file to the mock file system
+ Path partDir = new Path(conf.get("mapred.input.dir"));
+ OrcRecordUpdater writer = new OrcRecordUpdater(partDir,
+ new AcidOutputFormat.Options(conf).maximumTransactionId(10)
+ .writingBase(true).bucket(0).inspector(inspector));
+ for(int i=0; i < 10; ++i) {
+ writer.insert(10, new MyRow(i, 2 * i));
+ }
+ WriterImpl baseWriter = (WriterImpl) writer.getWriter();
+ writer.close(false);
+ ((MockOutputStream) baseWriter.getStream())
+ .setBlocks(new MockBlock("host0", "host1"));
+
+ // call getsplits
+ HiveInputFormat<?,?> inputFormat =
+ new HiveInputFormat<WritableComparable, Writable>();
+ InputSplit[] splits = inputFormat.getSplits(conf, 10);
+ assertEquals(1, splits.length);
+
+ try {
+ org.apache.hadoop.mapred.RecordReader<NullWritable, VectorizedRowBatch>
+ reader = inputFormat.getRecordReader(splits[0], conf, Reporter.NULL);
+ assertTrue("should throw here", false);
+ } catch (IOException ioe) {
+ assertEquals("java.io.IOException: Vectorization and ACID tables are incompatible.",
+ ioe.getMessage());
+ }
+ }
+
+ // test non-vectorized, non-acid, combine
+ @Test
+ public void testCombinationInputFormat() throws Exception {
+ // get the object inspector for MyRow
+ StructObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = (StructObjectInspector)
+ ObjectInspectorFactory.getReflectionObjectInspector(MyRow.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ JobConf conf = createMockExecutionEnvironment(workDir, new Path("mock:///"),
+ inspector, false);
+
+ // write the orc file to the mock file system
+ Path partDir = new Path(conf.get("mapred.input.dir"));
+ Writer writer =
+ OrcFile.createWriter(new Path(partDir, "0_0"),
+ OrcFile.writerOptions(conf).blockPadding(false)
+ .bufferSize(1024).inspector(inspector));
+ for(int i=0; i < 10; ++i) {
+ writer.addRow(new MyRow(i, 2*i));
+ }
+ writer.close();
+ MockOutputStream outputStream = (MockOutputStream) ((WriterImpl) writer).getStream();
+ outputStream.setBlocks(new MockBlock("host0", "host1"));
+ int length0 = outputStream.file.length;
+ writer =
+ OrcFile.createWriter(new Path(partDir, "1_0"),
+ OrcFile.writerOptions(conf).blockPadding(false)
+ .bufferSize(1024).inspector(inspector));
+ for(int i=10; i < 20; ++i) {
+ writer.addRow(new MyRow(i, 2*i));
+ }
+ writer.close();
+ outputStream = (MockOutputStream) ((WriterImpl) writer).getStream();
+ outputStream.setBlocks(new MockBlock("host1", "host2"));
+
+ // call getsplits
+ HiveInputFormat<?,?> inputFormat =
+ new CombineHiveInputFormat<WritableComparable, Writable>();
+ InputSplit[] splits = inputFormat.getSplits(conf, 1);
+ assertEquals(1, splits.length);
+ CombineHiveInputFormat.CombineHiveInputSplit split =
+ (CombineHiveInputFormat.CombineHiveInputSplit) splits[0];
+
+ // check split
+ assertEquals(2, split.getNumPaths());
+ assertEquals(partDir.toString() + "/0_0", split.getPath(0).toString());
+ assertEquals(partDir.toString() + "/1_0", split.getPath(1).toString());
+ assertEquals(length0, split.getLength(0));
+ assertEquals(outputStream.file.length, split.getLength(1));
+ assertEquals(0, split.getOffset(0));
+ assertEquals(0, split.getOffset(1));
+ // hadoop-1 gets 3 and hadoop-2 gets 0. *sigh*
+ // best answer would be 1.
+ assertTrue(3 >= split.getLocations().length);
+
+ // read split
+ org.apache.hadoop.mapred.RecordReader<CombineHiveKey, OrcStruct> reader =
+ inputFormat.getRecordReader(split, conf, Reporter.NULL);
+ CombineHiveKey key = reader.createKey();
+ OrcStruct value = reader.createValue();
+ for(int i=0; i < 20; i++) {
+ assertEquals(true, reader.next(key, value));
+ assertEquals(i, ((IntWritable) value.getFieldValue(0)).get());
+ }
+ assertEquals(false, reader.next(key, value));
+ }
+
+ // test non-vectorized, acid, combine
+ @Test
+ public void testCombinationInputFormatWithAcid() throws Exception {
+ // get the object inspector for MyRow
+ StructObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = (StructObjectInspector)
+ ObjectInspectorFactory.getReflectionObjectInspector(MyRow.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ JobConf conf = createMockExecutionEnvironment(workDir, new Path("mock:///"),
+ inspector, false);
+
+ // write the orc file to the mock file system
+ Path partDir = new Path(conf.get("mapred.input.dir"));
+ OrcRecordUpdater writer = new OrcRecordUpdater(partDir,
+ new AcidOutputFormat.Options(conf).maximumTransactionId(10)
+ .writingBase(true).bucket(0).inspector(inspector));
+ for(int i=0; i < 10; ++i) {
+ writer.insert(10, new MyRow(i, 2 * i));
+ }
+ WriterImpl baseWriter = (WriterImpl) writer.getWriter();
+ writer.close(false);
+ MockOutputStream outputStream = (MockOutputStream) baseWriter.getStream();
+ int length0 = outputStream.file.length;
+ writer = new OrcRecordUpdater(partDir,
+ new AcidOutputFormat.Options(conf).maximumTransactionId(10)
+ .writingBase(true).bucket(1).inspector(inspector));
+ for(int i=10; i < 20; ++i) {
+ writer.insert(10, new MyRow(i, 2*i));
+ }
+ baseWriter = (WriterImpl) writer.getWriter();
+ writer.close(false);
+ outputStream = (MockOutputStream) baseWriter.getStream();
+ outputStream.setBlocks(new MockBlock("host1", "host2"));
+
+ // call getsplits
+ HiveInputFormat<?,?> inputFormat =
+ new CombineHiveInputFormat<WritableComparable, Writable>();
+ try {
+ InputSplit[] splits = inputFormat.getSplits(conf, 1);
+ assertTrue("shouldn't reach here", false);
+ } catch (IOException ioe) {
+ assertEquals("CombineHiveInputFormat is incompatible"
+ + " with ACID tables. Please set hive.input.format=org.apache.hadoop"
+ + ".hive.ql.io.HiveInputFormat",
+ ioe.getMessage());
+ }
+ }
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewInputOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewInputOutputFormat.java?rev=1581977&r1=1581976&r2=1581977&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewInputOutputFormat.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewInputOutputFormat.java Wed Mar 26 18:14:37 2014
@@ -28,6 +28,7 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -191,7 +192,8 @@ public class TestNewInputOutputFormat {
Path outputFilePath = new Path(outputPath, "part-m-00000");
assertTrue(localFs.exists(outputFilePath));
- Reader reader = OrcFile.createReader(localFs, outputFilePath, conf);
+ Reader reader = OrcFile.createReader(outputFilePath,
+ OrcFile.readerOptions(conf).filesystem(localFs));
assertTrue(reader.getNumberOfRows() == rownum);
assertEquals(reader.getCompression(), CompressionKind.ZLIB);
StructObjectInspector soi =
@@ -205,7 +207,7 @@ public class TestNewInputOutputFormat {
.getPrimitiveCategory(),
PrimitiveObjectInspector.PrimitiveCategory.STRING);
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
Object row = rows.next(null);
IntWritable intWritable = (IntWritable)soi.getStructFieldData(row,
@@ -248,7 +250,8 @@ public class TestNewInputOutputFormat {
assertTrue(result);
Path outputFilePath = new Path(outputPath, "part-m-00000");
- Reader reader = OrcFile.createReader(localFs, outputFilePath, conf);
+ Reader reader = OrcFile.createReader(outputFilePath,
+ OrcFile.readerOptions(conf).filesystem(localFs));
assertEquals(reader.getCompression(), CompressionKind.SNAPPY);
localFs.delete(outputPath, true);
@@ -298,6 +301,7 @@ public class TestNewInputOutputFormat {
struct.add(0, key.get());
struct.add(1, count);
List<List<Object>> lastWordInfoList = new ArrayList<List<Object>>();
+ Collections.sort(lastwords);
for (String word : lastwords) {
List<Object> info = new ArrayList<Object>(2);
info.add(0, word);
@@ -351,9 +355,10 @@ public class TestNewInputOutputFormat {
assertTrue(result);
Path outputFilePath = new Path(outputPath, "part-r-00000");
- Reader reader = OrcFile.createReader(localFs, outputFilePath, conf);
+ Reader reader = OrcFile.createReader(outputFilePath,
+ OrcFile.readerOptions(conf).filesystem(localFs));
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
ObjectInspector orcOi = reader.getObjectInspector();
ObjectInspector stoi = TypeInfoUtils
.getStandardJavaObjectInspectorFromTypeInfo(OrcTestReducer3.typeInfo);
@@ -362,36 +367,36 @@ public class TestNewInputOutputFormat {
Object row = rows.next(null);
List<Object> converted = (List<Object>)converter.convert(row);
- assertEquals(converted.get(0), 1);
- assertEquals(converted.get(1), 1);
+ assertEquals(1, converted.get(0));
+ assertEquals(1, converted.get(1));
List<Object> list = (List<Object>)converted.get(2);
assertEquals(list.size(), 1);
- assertEquals(((List<Object>)list.get(0)).get(0), "saving");
- assertEquals(((List<Object>)list.get(0)).get(1), 6);
+ assertEquals("saving", ((List<Object>)list.get(0)).get(0));
+ assertEquals(6, ((List<Object>)list.get(0)).get(1));
Map<String, Integer> map = (Map<String, Integer>)converted.get(3);
assertEquals(map.size(), 1);
assertEquals(map.get("saving"), new Integer(1));
row = rows.next(null);
converted = (List<Object>)converter.convert(row);
- assertEquals(converted.get(0), 2);
- assertEquals(converted.get(1), 6);
+ assertEquals(2, converted.get(0));
+ assertEquals(6, converted.get(1));
list = (List<Object>)converted.get(2);
assertEquals(list.size(), 6);
- assertEquals(((List<Object>)list.get(0)).get(0), "plums");
- assertEquals(((List<Object>)list.get(0)).get(1), 5);
+ assertEquals("breakfast", ((List<Object>)list.get(0)).get(0));
+ assertEquals(9, ((List<Object>)list.get(0)).get(1));
map = (Map<String, Integer>)converted.get(3);
assertEquals(map.size(), 11);
assertEquals(map.get("the"), new Integer(2));
row = rows.next(null);
converted = (List<Object>)converter.convert(row);
- assertEquals(converted.get(0), 3);
- assertEquals(converted.get(1), 5);
+ assertEquals(3, converted.get(0));
+ assertEquals(5, converted.get(1));
list = (List<Object>)converted.get(2);
assertEquals(list.size(), 5);
- assertEquals(((List<Object>)list.get(0)).get(0), "eaten");
- assertEquals(((List<Object>)list.get(0)).get(1), 5);
+ assertEquals("cold", ((List<Object>)list.get(0)).get(0));
+ assertEquals(4, ((List<Object>)list.get(0)).get(1));
map = (Map<String, Integer>)converted.get(3);
assertEquals(map.size(), 13);
assertEquals(map.get("were"), new Integer(3));
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java?rev=1581977&r1=1581976&r2=1581977&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java Wed Mar 26 18:14:37 2014
@@ -20,13 +20,7 @@ package org.apache.hadoop.hive.ql.io.orc
import static junit.framework.Assert.assertEquals;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.charset.Charset;
import java.sql.Timestamp;
-import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@@ -65,37 +59,12 @@ public class TestNewIntegerEncoding {
}
}
- public List<Long> fetchData(String path) throws IOException {
- List<Long> input = new ArrayList<Long>();
- FileInputStream stream = new FileInputStream(new File(path));
- try {
- FileChannel fc = stream.getChannel();
- MappedByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
- /* Instead of using default, pass in a decoder. */
- String[] lines = Charset.defaultCharset().decode(bb).toString()
- .split("\n");
- for(String line : lines) {
- long val = 0;
- try {
- val = Long.parseLong(line);
- } catch (NumberFormatException e) {
- // for now lets ignore (assign 0)
- }
- input.add(val);
- }
- } finally {
- stream.close();
- }
- return input;
- }
-
Path workDir = new Path(System.getProperty("test.tmp.dir", "target"
+ File.separator + "test" + File.separator + "tmp"));
Configuration conf;
FileSystem fs;
Path testFilePath;
- String resDir = "ql/src/test/resources";
@Rule
public TestName testCaseName = new TestName();
@@ -128,8 +97,9 @@ public class TestNewIntegerEncoding {
writer.addRow(new Row(111, 1111L));
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
while (rows.hasNext()) {
Object row = rows.next(null);
assertEquals(new IntWritable(111), ((OrcStruct) row).getFieldValue(0));
@@ -162,8 +132,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -197,8 +168,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -228,8 +200,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -259,8 +232,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -290,8 +264,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -321,8 +296,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -351,8 +327,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -382,8 +359,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -413,8 +391,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -444,8 +423,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -478,8 +458,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -512,8 +493,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -554,8 +536,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -596,8 +579,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -638,8 +622,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -671,8 +656,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -706,8 +692,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -741,8 +728,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -775,8 +763,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -809,8 +798,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -843,8 +833,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -877,8 +868,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -908,8 +900,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -941,8 +934,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -987,8 +981,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -1036,8 +1031,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -1097,8 +1093,9 @@ public class TestNewIntegerEncoding {
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -1125,8 +1122,9 @@ public class TestNewIntegerEncoding {
writer.addRow(-5535739865598783616L);
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
Object row = rows.next(null);
assertEquals(-7486502418706614742L, ((LongWritable) row).get());
row = rows.next(row);
@@ -1164,8 +1162,9 @@ public class TestNewIntegerEncoding {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
int idx = 55555;
rows.seekToRow(idx);
while (rows.hasNext()) {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java?rev=1581977&r1=1581976&r2=1581977&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java Wed Mar 26 18:14:37 2014
@@ -41,6 +41,8 @@ import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
@@ -114,9 +116,7 @@ public class TestOrcFile {
MiddleStruct(InnerStruct... items) {
list.clear();
- for(InnerStruct item: items) {
- list.add(item);
- }
+ list.addAll(Arrays.asList(items));
}
}
@@ -171,9 +171,7 @@ public class TestOrcFile {
private static List<InnerStruct> list(InnerStruct... items) {
List<InnerStruct> result = new ArrayList<InnerStruct>();
- for(InnerStruct s: items) {
- result.add(s);
- }
+ result.addAll(Arrays.asList(items));
return result;
}
@@ -218,7 +216,7 @@ public class TestOrcFile {
public void openFileSystem () throws Exception {
conf = new Configuration();
if(zeroCopy) {
- conf.setBoolean(HIVE_ORC_ZEROCOPY.varname, zeroCopy);
+ conf.setBoolean(HiveConf.ConfVars.HIVE_ORC_ZEROCOPY.varname, zeroCopy);
}
fs = FileSystem.getLocal(conf);
testFilePath = new Path(workDir, "TestOrcFile." +
@@ -228,8 +226,10 @@ public class TestOrcFile {
@Test
public void testReadFormat_0_11() throws Exception {
- Path oldFilePath = new Path(HiveTestUtils.getFileFromClasspath("orc-file-11-format.orc"));
- Reader reader = OrcFile.createReader(fs, oldFilePath, conf);
+ Path oldFilePath =
+ new Path(HiveTestUtils.getFileFromClasspath("orc-file-11-format.orc"));
+ Reader reader = OrcFile.createReader(oldFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
int stripeCount = 0;
int rowCount = 0;
@@ -329,7 +329,7 @@ public class TestOrcFile {
.getStructFieldRef("decimal1").getFieldObjectInspector();
StringObjectInspector mk = (StringObjectInspector) ma
.getMapKeyObjectInspector();
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
Object row = rows.next(null);
assertNotNull(row);
// check the contents of the first row
@@ -505,7 +505,8 @@ public class TestOrcFile {
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows = reader.rows(null);
int idx = 0;
while (rows.hasNext()) {
@@ -532,7 +533,8 @@ public class TestOrcFile {
writer.addRow(new SimpleStruct(bytes(0,1,2,3,4,5), null));
writer.addRow(new SimpleStruct(null, "hi"));
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
// check the stats
ColumnStatistics[] stats = reader.getStatistics();
@@ -563,7 +565,7 @@ public class TestOrcFile {
getStructFieldRef("bytes1").getFieldObjectInspector();
StringObjectInspector st = (StringObjectInspector) readerInspector.
getStructFieldRef("string1").getFieldObjectInspector();
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
Object row = rows.next(null);
assertNotNull(row);
// check the contents of the first row
@@ -628,7 +630,8 @@ public class TestOrcFile {
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
Metadata metadata = reader.getMetadata();
int numStripes = metadata.getStripeStatistics().size();
assertEquals(3, numStripes);
@@ -640,9 +643,9 @@ public class TestOrcFile {
assertEquals(5000, ss2.getColumnStatistics()[0].getNumberOfValues());
assertEquals(1000, ss3.getColumnStatistics()[0].getNumberOfValues());
- assertEquals(5000, ((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getNumberOfValues());
- assertEquals(5000, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getNumberOfValues());
- assertEquals(1000, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getNumberOfValues());
+ assertEquals(5000, (ss1.getColumnStatistics()[1]).getNumberOfValues());
+ assertEquals(5000, (ss2.getColumnStatistics()[1]).getNumberOfValues());
+ assertEquals(1000, (ss3.getColumnStatistics()[1]).getNumberOfValues());
assertEquals(1, ((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getMinimum());
assertEquals(2, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getMinimum());
assertEquals(3, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getMinimum());
@@ -653,9 +656,9 @@ public class TestOrcFile {
assertEquals(10000, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getSum());
assertEquals(3000, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getSum());
- assertEquals(5000, ((StringColumnStatistics)ss1.getColumnStatistics()[2]).getNumberOfValues());
- assertEquals(5000, ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getNumberOfValues());
- assertEquals(1000, ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getNumberOfValues());
+ assertEquals(5000, (ss1.getColumnStatistics()[2]).getNumberOfValues());
+ assertEquals(5000, (ss2.getColumnStatistics()[2]).getNumberOfValues());
+ assertEquals(1000, (ss3.getColumnStatistics()[2]).getNumberOfValues());
assertEquals("one", ((StringColumnStatistics)ss1.getColumnStatistics()[2]).getMinimum());
assertEquals("two", ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getMinimum());
assertEquals("three", ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getMinimum());
@@ -666,7 +669,7 @@ public class TestOrcFile {
assertEquals(15000, ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getSum());
assertEquals(5000, ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getSum());
- RecordReaderImpl recordReader = (RecordReaderImpl) reader.rows(null);
+ RecordReaderImpl recordReader = (RecordReaderImpl) reader.rows();
OrcProto.RowIndex[] index = recordReader.readRowIndex(0);
assertEquals(3, index.length);
List<OrcProto.RowIndexEntry> items = index[1].getEntryList();
@@ -707,7 +710,8 @@ public class TestOrcFile {
list(inner(100000000, "cat"), inner(-100000, "in"), inner(1234, "hat")),
map(inner(5, "chani"), inner(1, "mauddib"))));
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
Metadata metadata = reader.getMetadata();
@@ -783,13 +787,9 @@ public class TestOrcFile {
getStructFieldRef("list").getFieldObjectInspector();
MapObjectInspector ma = (MapObjectInspector) readerInspector.
getStructFieldRef("map").getFieldObjectInspector();
- StructObjectInspector lc = (StructObjectInspector)
- li.getListElementObjectInspector();
StringObjectInspector mk = (StringObjectInspector)
ma.getMapKeyObjectInspector();
- StructObjectInspector mv = (StructObjectInspector)
- ma.getMapValueObjectInspector();
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
Object row = rows.next(null);
assertNotNull(row);
// check the contents of the first row
@@ -957,7 +957,8 @@ public class TestOrcFile {
writer.addRow(inner(x, y));
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
// check out the statistics
ColumnStatistics[] stats = reader.getStatistics();
@@ -1021,8 +1022,9 @@ public class TestOrcFile {
.compress(CompressionKind.NONE)
.bufferSize(100));
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- assertEquals(false, reader.rows(null).hasNext());
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ assertEquals(false, reader.rows().hasNext());
assertEquals(CompressionKind.NONE, reader.getCompression());
assertEquals(0, reader.getNumberOfRows());
assertEquals(0, reader.getCompressionSize());
@@ -1058,7 +1060,8 @@ public class TestOrcFile {
null, null, null, null));
writer.addUserMetadata("clobber", byteBuf(5,7,11,13,17,19));
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(byteBuf(5,7,11,13,17,19), reader.getMetadataValue("clobber"));
assertEquals(byteBuf(1,2,3,4,5,6,7,-1,-2,127,-128),
reader.getMetadataValue("my.meta"));
@@ -1152,7 +1155,7 @@ public class TestOrcFile {
if ((i & 1) == 0) {
union.set((byte) 0, new IntWritable(i*i));
} else {
- union.set((byte) 1, new Text(new Integer(i*i).toString()));
+ union.set((byte) 1, new Text(Integer.toString(i * i)));
}
value = HiveDecimal.create(new BigInteger(64, rand),
rand.nextInt(18));
@@ -1176,7 +1179,8 @@ public class TestOrcFile {
union.set((byte) 0, new IntWritable(138));
writer.addRow(row);
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(false, reader.getMetadataKeys().iterator().hasNext());
assertEquals(5309, reader.getNumberOfRows());
DecimalColumnStatistics stats =
@@ -1202,7 +1206,7 @@ public class TestOrcFile {
assertEquals(reader.getNumberOfRows(), rowCount);
assertEquals(2, stripeCount);
assertEquals(reader.getContentLength(), currentOffset);
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
assertEquals(0, rows.getRowNumber());
assertEquals(0.0, rows.getProgress(), 0.000001);
assertEquals(true, rows.hasNext());
@@ -1255,7 +1259,7 @@ public class TestOrcFile {
assertEquals(new IntWritable(i*i), union.getObject());
} else {
assertEquals(1, union.getTag());
- assertEquals(new Text(new Integer(i*i).toString()), union.getObject());
+ assertEquals(new Text(Integer.toString(i * i)), union.getObject());
}
assertEquals(HiveDecimal.create(new BigInteger(64, rand),
rand.nextInt(18)), row.getFieldValue(2));
@@ -1307,8 +1311,9 @@ public class TestOrcFile {
Integer.toHexString(rand.nextInt())));
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
- RecordReader rows = reader.rows(null);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
rand = new Random(12);
OrcStruct row = null;
for(int i=0; i < 10000; ++i) {
@@ -1350,13 +1355,14 @@ public class TestOrcFile {
}
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(50000, reader.getNumberOfRows());
assertEquals(0, reader.getRowIndexStride());
StripeInformation stripe = reader.getStripes().iterator().next();
assertEquals(true, stripe.getDataLength() != 0);
assertEquals(0, stripe.getIndexLength());
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
rand = new Random(24);
OrcStruct row = null;
for(int i=0; i < 10000; ++i) {
@@ -1413,10 +1419,124 @@ public class TestOrcFile {
byteValues, words, i));
}
writer.close();
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ assertEquals(COUNT, reader.getNumberOfRows());
+ RecordReader rows = reader.rows();
+ OrcStruct row = null;
+ for(int i=COUNT-1; i >= 0; --i) {
+ rows.seekToRow(i);
+ row = (OrcStruct) rows.next(row);
+ BigRow expected = createRandomRow(intValues, doubleValues,
+ stringValues, byteValues, words, i);
+ assertEquals(expected.boolean1.booleanValue(),
+ ((BooleanWritable) row.getFieldValue(0)).get());
+ assertEquals(expected.byte1.byteValue(),
+ ((ByteWritable) row.getFieldValue(1)).get());
+ assertEquals(expected.short1.shortValue(),
+ ((ShortWritable) row.getFieldValue(2)).get());
+ assertEquals(expected.int1.intValue(),
+ ((IntWritable) row.getFieldValue(3)).get());
+ assertEquals(expected.long1.longValue(),
+ ((LongWritable) row.getFieldValue(4)).get());
+ assertEquals(expected.float1,
+ ((FloatWritable) row.getFieldValue(5)).get(), 0.0001);
+ assertEquals(expected.double1,
+ ((DoubleWritable) row.getFieldValue(6)).get(), 0.0001);
+ assertEquals(expected.bytes1, row.getFieldValue(7));
+ assertEquals(expected.string1, row.getFieldValue(8));
+ List<InnerStruct> expectedList = expected.middle.list;
+ List<OrcStruct> actualList =
+ (List<OrcStruct>) ((OrcStruct) row.getFieldValue(9)).getFieldValue(0);
+ compareList(expectedList, actualList);
+ compareList(expected.list, (List<OrcStruct>) row.getFieldValue(10));
+ }
+ rows.close();
+ Iterator<StripeInformation> stripeIterator =
+ reader.getStripes().iterator();
+ long offsetOfStripe2 = 0;
+ long offsetOfStripe4 = 0;
+ long lastRowOfStripe2 = 0;
+ for(int i = 0; i < 5; ++i) {
+ StripeInformation stripe = stripeIterator.next();
+ if (i < 2) {
+ lastRowOfStripe2 += stripe.getNumberOfRows();
+ } else if (i == 2) {
+ offsetOfStripe2 = stripe.getOffset();
+ lastRowOfStripe2 += stripe.getNumberOfRows() - 1;
+ } else if (i == 4) {
+ offsetOfStripe4 = stripe.getOffset();
+ }
+ }
+ boolean[] columns = new boolean[reader.getStatistics().length];
+ columns[5] = true; // long colulmn
+ columns[9] = true; // text column
+ rows = reader.rowsOptions(new Reader.Options()
+ .range(offsetOfStripe2, offsetOfStripe4 - offsetOfStripe2)
+ .include(columns));
+ rows.seekToRow(lastRowOfStripe2);
+ for(int i = 0; i < 2; ++i) {
+ row = (OrcStruct) rows.next(row);
+ BigRow expected = createRandomRow(intValues, doubleValues,
+ stringValues, byteValues, words,
+ (int) (lastRowOfStripe2 + i));
+
+ assertEquals(expected.long1.longValue(),
+ ((LongWritable) row.getFieldValue(4)).get());
+ assertEquals(expected.string1, row.getFieldValue(8));
+ }
+ rows.close();
+ }
+
+ @Test
+ public void testZeroCopySeek() throws Exception {
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector
+ (BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ Writer writer = OrcFile.createWriter(testFilePath,
+ OrcFile.writerOptions(conf)
+ .inspector(inspector)
+ .stripeSize(200000)
+ .bufferSize(65536)
+ .rowIndexStride(1000));
+ Random rand = new Random(42);
+ final int COUNT=32768;
+ long[] intValues= new long[COUNT];
+ double[] doubleValues = new double[COUNT];
+ String[] stringValues = new String[COUNT];
+ BytesWritable[] byteValues = new BytesWritable[COUNT];
+ String[] words = new String[128];
+ for(int i=0; i < words.length; ++i) {
+ words[i] = Integer.toHexString(rand.nextInt());
+ }
+ for(int i=0; i < COUNT/2; ++i) {
+ intValues[2*i] = rand.nextLong();
+ intValues[2*i+1] = intValues[2*i];
+ stringValues[2*i] = words[rand.nextInt(words.length)];
+ stringValues[2*i+1] = stringValues[2*i];
+ }
+ for(int i=0; i < COUNT; ++i) {
+ doubleValues[i] = rand.nextDouble();
+ byte[] buf = new byte[20];
+ rand.nextBytes(buf);
+ byteValues[i] = new BytesWritable(buf);
+ }
+ for(int i=0; i < COUNT; ++i) {
+ writer.addRow(createRandomRow(intValues, doubleValues, stringValues,
+ byteValues, words, i));
+ }
+ writer.close();
writer = null;
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(COUNT, reader.getNumberOfRows());
- RecordReader rows = reader.rows(null);
+ /* enable zero copy record reader */
+ Configuration conf = new Configuration();
+ HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_ZEROCOPY, true);
+ RecordReader rows = reader.rows();
+ /* all tests are identical to the other seek() tests */
OrcStruct row = null;
for(int i=COUNT-1; i >= 0; --i) {
rows.seekToRow(i);
@@ -1465,8 +1585,10 @@ public class TestOrcFile {
boolean[] columns = new boolean[reader.getStatistics().length];
columns[5] = true; // long colulmn
columns[9] = true; // text column
- rows = reader.rows(offsetOfStripe2, offsetOfStripe4 - offsetOfStripe2,
- columns);
+ /* use zero copy record reader */
+ rows = reader.rowsOptions(new Reader.Options()
+ .range(offsetOfStripe2, offsetOfStripe4 - offsetOfStripe2)
+ .include(columns));
rows.seekToRow(lastRowOfStripe2);
for(int i = 0; i < 2; ++i) {
row = (OrcStruct) rows.next(row);
@@ -1484,7 +1606,8 @@ public class TestOrcFile {
private void compareInner(InnerStruct expect,
OrcStruct actual) throws Exception {
if (expect == null || actual == null) {
- assertEquals(expect, actual);
+ assertEquals(null, expect);
+ assertEquals(null, actual);
} else {
assertEquals(expect.int1, ((IntWritable) actual.getFieldValue(0)).get());
assertEquals(expect.string1, actual.getFieldValue(1));
@@ -1581,7 +1704,8 @@ public class TestOrcFile {
}
writer.close();
assertEquals(null, memory.path);
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
int i = 0;
for(StripeInformation stripe: reader.getStripes()) {
i += 1;
@@ -1606,7 +1730,8 @@ public class TestOrcFile {
writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
}
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(3500, reader.getNumberOfRows());
SearchArgument sarg = SearchArgument.FACTORY.newBuilder()
@@ -1617,9 +1742,10 @@ public class TestOrcFile {
.lessThan("int1", 600000)
.end()
.build();
- RecordReader rows = reader.rows(0L, Long.MAX_VALUE,
- new boolean[]{true, true, true}, sarg,
- new String[]{null, "int1", "string1"});
+ RecordReader rows = reader.rowsOptions(new Reader.Options()
+ .range(0L, Long.MAX_VALUE)
+ .include(new boolean[]{true, true, true})
+ .searchArgument(sarg, new String[]{null, "int1", "string1"}));
assertEquals(1000L, rows.getRowNumber());
OrcStruct row = null;
for(int i=1000; i < 2000; ++i) {
@@ -1637,9 +1763,10 @@ public class TestOrcFile {
.lessThan("int1", 0)
.end()
.build();
- rows = reader.rows(0L, Long.MAX_VALUE,
- new boolean[]{true, true, true}, sarg,
- new String[]{null, "int1", "string1"});
+ rows = reader.rowsOptions(new Reader.Options()
+ .range(0L, Long.MAX_VALUE)
+ .include(new boolean[]{true, true, true})
+ .searchArgument(sarg, new String[]{null, "int1", "string1"}));
assertEquals(3500L, rows.getRowNumber());
assertTrue(!rows.hasNext());
@@ -1652,9 +1779,10 @@ public class TestOrcFile {
.end()
.end()
.build();
- rows = reader.rows(0L, Long.MAX_VALUE,
- new boolean[]{true, true, true}, sarg,
- new String[]{null, "int1", "string1"});
+ rows = reader.rowsOptions(new Reader.Options()
+ .range(0L, Long.MAX_VALUE)
+ .include(new boolean[]{true, true, true})
+ .searchArgument(sarg, new String[]{null, "int1", "string1"}));
row = null;
for(int i=0; i < 1000; ++i) {
assertTrue(rows.hasNext());
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java?rev=1581977&r1=1581976&r2=1581977&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java Wed Mar 26 18:14:37 2014
@@ -108,7 +108,8 @@ public class TestOrcNullOptimization {
Lists.newArrayList(new InnerStruct(100))));
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
// check the stats
ColumnStatistics[] stats = reader.getStatistics();
assertEquals(20000, reader.getNumberOfRows());
@@ -123,8 +124,7 @@ public class TestOrcNullOptimization {
assertEquals("a", ((StringColumnStatistics) stats[2]).getMaximum());
assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum());
- assertEquals(19998,
- ((StringColumnStatistics) stats[2]).getNumberOfValues());
+ assertEquals(19998, stats[2].getNumberOfValues());
assertEquals("count: 19998 min: a max: a sum: 19998",
stats[2].toString());
@@ -136,7 +136,7 @@ public class TestOrcNullOptimization {
assertEquals("struct<a:int,b:string,c:boolean,list:array<struct<z:int>>>",
readerInspector.getTypeName());
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
List<Boolean> expected = Lists.newArrayList();
for (StripeInformation sinfo : reader.getStripes()) {
@@ -212,7 +212,8 @@ public class TestOrcNullOptimization {
Lists.newArrayList(new InnerStruct(100))));
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
// check the stats
ColumnStatistics[] stats = reader.getStatistics();
assertEquals(20000, reader.getNumberOfRows());
@@ -227,8 +228,7 @@ public class TestOrcNullOptimization {
assertEquals("b", ((StringColumnStatistics) stats[2]).getMaximum());
assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum());
- assertEquals(20000,
- ((StringColumnStatistics) stats[2]).getNumberOfValues());
+ assertEquals(20000, stats[2].getNumberOfValues());
assertEquals("count: 20000 min: a max: b sum: 20000",
stats[2].toString());
@@ -240,7 +240,7 @@ public class TestOrcNullOptimization {
assertEquals("struct<a:int,b:string,c:boolean,list:array<struct<z:int>>>",
readerInspector.getTypeName());
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
// none of the stripes will have PRESENT stream
List<Boolean> expected = Lists.newArrayList();
@@ -313,7 +313,8 @@ public class TestOrcNullOptimization {
Lists.newArrayList(new InnerStruct(100))));
writer.close();
- Reader reader = OrcFile.createReader(fs, testFilePath, conf);
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
// check the stats
ColumnStatistics[] stats = reader.getStatistics();
assertEquals(8, reader.getNumberOfRows());
@@ -328,7 +329,7 @@ public class TestOrcNullOptimization {
assertEquals("h", ((StringColumnStatistics) stats[2]).getMaximum());
assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum());
- assertEquals(7, ((StringColumnStatistics) stats[2]).getNumberOfValues());
+ assertEquals(7, stats[2].getNumberOfValues());
assertEquals("count: 7 min: a max: h sum: 7",
stats[2].toString());
@@ -340,7 +341,7 @@ public class TestOrcNullOptimization {
assertEquals("struct<a:int,b:string,c:boolean,list:array<struct<z:int>>>",
readerInspector.getTypeName());
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
// only the last strip will have PRESENT stream
List<Boolean> expected = Lists.newArrayList();
for (StripeInformation sinfo : reader.getStripes()) {