You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2017/08/31 22:27:15 UTC
[1/2] orc git commit: ORC-231: Configurable capability to overwrite
the file if it exists. [Forced Update!]
Repository: orc
Updated Branches:
refs/heads/orc-162 0a2fb7065 -> 14cfddfa0 (forced update)
ORC-231: Configurable capability to overwrite the file if it exists.
The default value of this flag is false and this change is backward
compatible.
Fixes #162
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/22c6c5bf
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/22c6c5bf
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/22c6c5bf
Branch: refs/heads/orc-162
Commit: 22c6c5bf2463c6fdd7ecc546e669c4ea0cdeba3d
Parents: 1b30e32
Author: Ajay Yadava <aj...@apache.org>
Authored: Mon Aug 21 23:49:54 2017 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Aug 29 12:43:53 2017 -0700
----------------------------------------------------------------------
java/core/src/java/org/apache/orc/OrcConf.java | 4 +-
java/core/src/java/org/apache/orc/OrcFile.java | 15 +++++
.../org/apache/orc/impl/PhysicalFsWriter.java | 2 +-
.../org/apache/orc/impl/TestWriterImpl.java | 71 ++++++++++++++++++++
4 files changed, 90 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/22c6c5bf/java/core/src/java/org/apache/orc/OrcConf.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/OrcConf.java b/java/core/src/java/org/apache/orc/OrcConf.java
index 10d4228..35328da 100644
--- a/java/core/src/java/org/apache/orc/OrcConf.java
+++ b/java/core/src/java/org/apache/orc/OrcConf.java
@@ -146,7 +146,9 @@ public enum OrcConf {
ROWS_BETWEEN_CHECKS("orc.rows.between.memory.checks", "orc.rows.between.memory.checks", 5000,
"How often should MemoryManager check the memory sizes? Measured in rows\n" +
"added to all of the writers. Valid range is [1,10000] and is primarily meant for" +
- "n\testing. Setting this too low may negatively affect performance.")
+ "n\testing. Setting this too low may negatively affect performance."),
+ OVERWRITE_OUTPUT_FILE("orc.overwrite.output.file", "orc.overwrite.output.file", false,
+ "A boolean flag to enable overwriting of the output file if it already exists.\n")
;
private final String attribute;
http://git-wip-us.apache.org/repos/asf/orc/blob/22c6c5bf/java/core/src/java/org/apache/orc/OrcFile.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/OrcFile.java b/java/core/src/java/org/apache/orc/OrcFile.java
index 99f22dd..5375efe 100644
--- a/java/core/src/java/org/apache/orc/OrcFile.java
+++ b/java/core/src/java/org/apache/orc/OrcFile.java
@@ -384,10 +384,12 @@ public class OrcFile {
private BloomFilterVersion bloomFilterVersion;
private PhysicalWriter physicalWriter;
private WriterVersion writerVersion = CURRENT_WRITER;
+ private boolean overwrite;
protected WriterOptions(Properties tableProperties, Configuration conf) {
configuration = conf;
memoryManagerValue = getStaticMemoryManager(conf);
+ overwrite = OrcConf.OVERWRITE_OUTPUT_FILE.getBoolean(tableProperties, conf);
stripeSizeValue = OrcConf.STRIPE_SIZE.getLong(tableProperties, conf);
blockSizeValue = OrcConf.BLOCK_SIZE.getLong(tableProperties, conf);
rowIndexStrideValue =
@@ -445,6 +447,15 @@ public class OrcFile {
}
/**
+ * If the output file already exists, should it be overwritten?
+ * If it is not provided, write operation will fail if the file already exists.
+ */
+ public WriterOptions overwrite(boolean value) {
+ overwrite = value;
+ return this;
+ }
+
+ /**
* Set the stripe size for the file. The writer stores the contents of the
* stripe in memory until this memory limit is reached and the stripe
* is flushed to the HDFS file and the next stripe started.
@@ -631,6 +642,10 @@ public class OrcFile {
return bloomFilterColumns;
}
+ public boolean getOverwrite() {
+ return overwrite;
+ }
+
public FileSystem getFileSystem() {
return fileSystemValue;
}
http://git-wip-us.apache.org/repos/asf/orc/blob/22c6c5bf/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java b/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
index 1769182..06e762f 100644
--- a/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
+++ b/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
@@ -88,7 +88,7 @@ public class PhysicalFsWriter implements PhysicalWriter {
LOG.info("ORC writer created for path: {} with stripeSize: {} blockSize: {}" +
" compression: {} bufferSize: {}", path, defaultStripeSize, blockSize,
compress, bufferSize);
- rawWriter = fs.create(path, false, HDFS_BUFFER_SIZE,
+ rawWriter = fs.create(path, opts.getOverwrite(), HDFS_BUFFER_SIZE,
fs.getDefaultReplication(path), blockSize);
codec = OrcCodecPool.getCodec(compress);
writer = new OutStream("metadata", bufferSize, codec,
http://git-wip-us.apache.org/repos/asf/orc/blob/22c6c5bf/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java b/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
new file mode 100644
index 0000000..22bb1d3
--- /dev/null
+++ b/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.orc.OrcConf;
+import org.apache.orc.OrcFile;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestWriterImpl {
+
+ Path workDir = new Path(System.getProperty("test.tmp.dir"));
+ Configuration conf;
+ FileSystem fs;
+ Path testFilePath;
+ TypeDescription schema;
+
+ @Before
+ public void openFileSystem() throws Exception {
+ conf = new Configuration();
+ fs = FileSystem.getLocal(conf);
+ fs.setWorkingDirectory(workDir);
+ testFilePath = new Path("testWriterImpl.orc");
+ fs.create(testFilePath, true);
+ schema = TypeDescription.fromString("struct<x:int,y:int>");
+ }
+
+ @After
+ public void deleteTestFile() throws Exception {
+ fs.delete(testFilePath, false);
+ }
+
+ @Test(expected = FileAlreadyExistsException.class)
+ public void testDefaultOverwriteFlagForWriter() throws Exception {
+ // default value of the overwrite flag is false, so this should fail
+ Writer w = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).setSchema(schema));
+ w.close();
+ }
+
+ @Test
+ public void testOverriddenOverwriteFlagForWriter() throws Exception {
+ // overriding the flag should result in a successful write (no exception)
+ conf.set(OrcConf.OVERWRITE_OUTPUT_FILE.getAttribute(), "true");
+ Writer w = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).setSchema(schema));
+ w.close();
+ }
+}
[2/2] orc git commit: ORC-162. Handle 0 byte files as empty ORC files.
Posted by om...@apache.org.
ORC-162. Handle 0 byte files as empty ORC files.
Fixes #163
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/14cfddfa
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/14cfddfa
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/14cfddfa
Branch: refs/heads/orc-162
Commit: 14cfddfa084bbd40200c3a40cbc3f1cdfb7e40aa
Parents: 22c6c5b
Author: Owen O'Malley <om...@apache.org>
Authored: Tue Aug 29 11:07:39 2017 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Thu Aug 31 15:26:05 2017 -0700
----------------------------------------------------------------------
examples/zero.orc | 0
.../java/org/apache/orc/TypeDescription.java | 11 +++--
.../java/org/apache/orc/impl/ReaderImpl.java | 44 ++++++++++++++++++--
.../src/test/org/apache/orc/TestReader.java | 5 ++-
.../test/org/apache/orc/TestVectorOrcFile.java | 26 +++++++++++-
.../org/apache/orc/mapred/OrcInputFormat.java | 24 +++++++++++
.../apache/orc/mapreduce/OrcInputFormat.java | 15 +++++++
7 files changed, 116 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/14cfddfa/examples/zero.orc
----------------------------------------------------------------------
diff --git a/examples/zero.orc b/examples/zero.orc
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/orc/blob/14cfddfa/java/core/src/java/org/apache/orc/TypeDescription.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/TypeDescription.java b/java/core/src/java/org/apache/orc/TypeDescription.java
index b0857f0..7f4d241 100644
--- a/java/core/src/java/org/apache/orc/TypeDescription.java
+++ b/java/core/src/java/org/apache/orc/TypeDescription.java
@@ -315,12 +315,17 @@ public class TypeDescription
static void parseStruct(TypeDescription type, StringPosition source) {
requireChar(source, '<');
- do {
+ boolean needComma = false;
+ while (!consumeChar(source, '>')) {
+ if (needComma) {
+ requireChar(source, ',');
+ } else {
+ needComma = true;
+ }
String fieldName = parseName(source);
requireChar(source, ':');
type.addField(fieldName, parseType(source));
- } while (consumeChar(source, ','));
- requireChar(source, '>');
+ }
}
static TypeDescription parseType(StringPosition source) {
http://git-wip-us.apache.org/repos/asf/orc/blob/14cfddfa/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
index 15d3c3f..130048b 100644
--- a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
@@ -491,6 +491,40 @@ public class ReaderImpl implements Reader {
return new OrcTail(fileTailBuilder.build(), buffer.slice(), modificationTime);
}
+ /**
+ * Build a virtual OrcTail for empty files.
+ * @return a new OrcTail
+ */
+ OrcTail buildEmptyTail() {
+ OrcProto.PostScript.Builder postscript = OrcProto.PostScript.newBuilder();
+ OrcFile.Version version = OrcFile.Version.CURRENT;
+ postscript.setMagic(OrcFile.MAGIC)
+ .setCompression(OrcProto.CompressionKind.NONE)
+ .setFooterLength(0)
+ .addVersion(version.getMajor())
+ .addVersion(version.getMinor())
+ .setMetadataLength(0)
+ .setWriterVersion(OrcFile.CURRENT_WRITER.getId());
+
+ // Use a struct with no fields
+ OrcProto.Type.Builder struct = OrcProto.Type.newBuilder();
+ struct.setKind(OrcProto.Type.Kind.STRUCT);
+
+ OrcProto.Footer.Builder footer = OrcProto.Footer.newBuilder();
+ footer.setHeaderLength(0)
+ .setContentLength(0)
+ .addTypes(struct)
+ .setNumberOfRows(0)
+ .setRowIndexStride(0);
+
+ OrcProto.FileTail.Builder result = OrcProto.FileTail.newBuilder();
+ result.setFooter(footer);
+ result.setPostscript(postscript);
+ result.setFileLength(0);
+ result.setPostscriptLength(0);
+ return new OrcTail(result.build(), null);
+ }
+
protected OrcTail extractFileTail(FileSystem fs, Path path,
long maxFileLength) throws IOException {
FSDataInputStream file = fs.open(path);
@@ -509,9 +543,13 @@ public class ReaderImpl implements Reader {
size = maxFileLength;
modificationTime = -1;
}
- // Anything lesser than MAGIC header cannot be valid (valid ORC file is actually around 45 bytes, this is
- // more conservative)
- if (size <= OrcFile.MAGIC.length()) {
+ if (size == 0) {
+ // Hive often creates empty files (including ORC) and has an
+ // optimization to create a 0 byte file as an empty ORC file.
+ return buildEmptyTail();
+ } else if (size <= OrcFile.MAGIC.length()) {
+ // Anything smaller than MAGIC header cannot be valid (valid ORC files
+ // are actually around 40 bytes, this is more conservative)
throw new FileFormatException("Not a valid ORC file " + path
+ " (maxFileLength= " + maxFileLength + ")");
}
http://git-wip-us.apache.org/repos/asf/orc/blob/14cfddfa/java/core/src/test/org/apache/orc/TestReader.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/TestReader.java b/java/core/src/test/org/apache/orc/TestReader.java
index fc4ebdc..61fd6e2 100644
--- a/java/core/src/test/org/apache/orc/TestReader.java
+++ b/java/core/src/test/org/apache/orc/TestReader.java
@@ -49,13 +49,14 @@ public class TestReader {
fs.delete(testFilePath, false);
}
- @Test(expected=FileFormatException.class)
+ @Test
public void testReadZeroLengthFile() throws Exception {
FSDataOutputStream fout = fs.create(testFilePath);
fout.close();
assertEquals(0, fs.getFileStatus(testFilePath).getLen());
- OrcFile.createReader(testFilePath,
+ Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
+ assertEquals(0, reader.getNumberOfRows());
}
@Test(expected=FileFormatException.class)
http://git-wip-us.apache.org/repos/asf/orc/blob/14cfddfa/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
index 9373216..6fe132b 100644
--- a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
+++ b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -3250,4 +3250,28 @@ public class TestVectorOrcFile {
assertEquals(fromString("baz"), reader.getMetadataValue("c"));
assertEquals(fromString("bat"), reader.getMetadataValue("d"));
}
+
+ Path exampleDir = new Path(System.getProperty("example.dir",
+ "../../examples/"));
+
+ @Test
+ public void testZeroByteOrcFile() throws Exception {
+ Path zeroFile = new Path(exampleDir, "zero.orc");
+ Reader reader = OrcFile.createReader(zeroFile, OrcFile.readerOptions(conf));
+ assertEquals(0, reader.getNumberOfRows());
+ assertEquals("struct<>", reader.getSchema().toString());
+ assertEquals(CompressionKind.NONE, reader.getCompressionKind());
+ assertEquals(0, reader.getRawDataSize());
+ assertEquals(0, reader.getRowIndexStride());
+ assertEquals(0, reader.getCompressionSize());
+ assertEquals(0, reader.getMetadataSize());
+ assertEquals(OrcFile.Version.CURRENT, reader.getFileVersion());
+ assertEquals(0, reader.getStripes().size());
+ assertEquals(0, reader.getStatistics().length);
+ assertEquals(0, reader.getMetadataKeys().size());
+ assertEquals(OrcFile.CURRENT_WRITER, reader.getWriterVersion());
+ VectorizedRowBatch batch =
+ TypeDescription.fromString("struct<>").createRowBatch();
+ assertEquals(false, reader.rows().nextBatch(batch));
+ }
}
http://git-wip-us.apache.org/repos/asf/orc/blob/14cfddfa/java/mapreduce/src/java/org/apache/orc/mapred/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcInputFormat.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcInputFormat.java
index 9e011de..c6975d8 100644
--- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcInputFormat.java
+++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcInputFormat.java
@@ -24,6 +24,7 @@ import com.esotericsoftware.kryo.io.Output;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl;
import org.apache.hadoop.io.WritableComparable;
@@ -32,6 +33,7 @@ import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.RecordReader;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.io.NullWritable;
@@ -151,4 +153,26 @@ public class OrcInputFormat<V extends WritableComparable>
return new OrcMapredRecordReader<>(file, buildOptions(conf,
file, split.getStart(), split.getLength()));
}
+
+ /**
+ * Filter out the 0 byte files, so that we don't generate splits for the
+ * empty ORC files.
+ * @param job the job configuration
+ * @return a list of files that need to be read
+ * @throws IOException
+ */
+ protected FileStatus[] listStatus(JobConf job) throws IOException {
+ FileStatus[] result = super.listStatus(job);
+ List<FileStatus> ok = new ArrayList<>(result.length);
+ for(FileStatus stat: result) {
+ if (stat.getLen() != 0) {
+ ok.add(stat);
+ }
+ }
+ if (ok.size() == result.length) {
+ return result;
+ } else {
+ return ok.toArray(new FileStatus[ok.size()]);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/orc/blob/14cfddfa/java/mapreduce/src/java/org/apache/orc/mapreduce/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/java/org/apache/orc/mapreduce/OrcInputFormat.java b/java/mapreduce/src/java/org/apache/orc/mapreduce/OrcInputFormat.java
index 9427e78..c588c6d 100644
--- a/java/mapreduce/src/java/org/apache/orc/mapreduce/OrcInputFormat.java
+++ b/java/mapreduce/src/java/org/apache/orc/mapreduce/OrcInputFormat.java
@@ -19,9 +19,11 @@
package org.apache.orc.mapreduce;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -29,6 +31,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.io.NullWritable;
import org.apache.orc.OrcConf;
@@ -68,4 +72,15 @@ public class OrcInputFormat<V extends WritableComparable>
org.apache.orc.mapred.OrcInputFormat.buildOptions(conf,
file, split.getStart(), split.getLength()));
}
+
+ protected List<FileStatus> listStatus(JobContext job) throws IOException {
+ List<FileStatus> complete = super.listStatus(job);
+ List<FileStatus> result = new ArrayList<>(complete.size());
+ for(FileStatus stat: complete) {
+ if (stat.getLen() != 0) {
+ result.add(stat);
+ }
+ }
+ return result;
+ }
}