You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by oz...@apache.org on 2016/08/10 16:38:08 UTC
[1/3] nifi git commit: NIFI-1663: Add ConvertAvroToORC processor
Repository: nifi
Updated Branches:
refs/heads/master bb2431270 -> d9720239f
http://git-wip-us.apache.org/repos/asf/nifi/blob/d9720239/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/ConvertAvroToORC.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/ConvertAvroToORC.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/ConvertAvroToORC.java
new file mode 100644
index 0000000..6100bd1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/ConvertAvroToORC.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.hive;
+
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
+import org.apache.hadoop.hive.ql.io.orc.OrcFlowFileWriter;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.util.hive.HiveJdbcCommon;
+import org.apache.nifi.util.hive.HiveUtils;
+import org.apache.hadoop.hive.ql.io.orc.NiFiOrcUtils;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * The ConvertAvroToORC processor takes an Avro-formatted flow file as input and converts it into ORC format.
+ */
+@SideEffectFree
+@SupportsBatching
+@Tags({"avro", "orc", "hive", "convert"})
+@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
+@CapabilityDescription("Converts an Avro record into ORC file format. This processor provides a direct mapping of an Avro record to an ORC record, such "
+ + "that the resulting ORC file will have the same hierarchical structure as the Avro document. If an incoming FlowFile contains a stream of "
+ + "multiple Avro records, the resultant FlowFile will contain a ORC file containing all of the Avro records. If an incoming FlowFile does "
+ + "not contain any records, an empty ORC file is the output.")
+@WritesAttributes({
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime type to application/octet-stream"),
+ @WritesAttribute(attribute = "filename", description = "Sets the filename to the existing filename with the extension replaced by / added to by .orc"),
+ @WritesAttribute(attribute = "record.count", description = "Sets the number of records in the ORC file."),
+ @WritesAttribute(attribute = "hive.ddl", description = "Creates a partial Hive DDL statement for creating a table in Hive from this ORC file. "
+ + "This can be used in ReplaceText for setting the content to the DDL. To make it valid DDL, add \"LOCATION '<path_to_orc_file_in_hdfs>'\", where "
+ + "the path is the directory that contains this ORC file on HDFS. For example, ConvertAvroToORC can send flow files to a PutHDFS processor to send the file to "
+ + "HDFS, then to a ReplaceText to set the content to this DDL (plus the LOCATION clause as described), then to PutHiveQL processor to create the table "
+ + "if it doesn't exist.")
+})
+public class ConvertAvroToORC extends AbstractProcessor {
+
+ // Attributes
+ public static final String ORC_MIME_TYPE = "application/octet-stream";
+ public static final String HIVE_DDL_ATTRIBUTE = "hive.ddl";
+ public static final String RECORD_COUNT_ATTRIBUTE = "record.count";
+
+
+ // Properties
+ public static final PropertyDescriptor ORC_CONFIGURATION_RESOURCES = new PropertyDescriptor.Builder()
+ .name("orc-config-resources")
+ .displayName("ORC Configuration Resources")
+ .description("A file or comma separated list of files which contains the ORC configuration (hive-site.xml, e.g.). Without this, Hadoop "
+ + "will search the classpath for a 'hive-site.xml' file or will revert to a default configuration. Please see the ORC documentation for more details.")
+ .required(false).addValidator(HiveUtils.createMultipleFilesExistValidator()).build();
+
+ public static final PropertyDescriptor STRIPE_SIZE = new PropertyDescriptor.Builder()
+ .name("orc-stripe-size")
+ .displayName("Stripe Size")
+ .description("The size of the memory buffer (in bytes) for writing stripes to an ORC file")
+ .required(true)
+ .addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
+ .defaultValue("100 KB")
+ .build();
+
+ public static final PropertyDescriptor BUFFER_SIZE = new PropertyDescriptor.Builder()
+ .name("orc-buffer-size")
+ .displayName("Buffer Size")
+ .description("The maximum size of the memory buffers (in bytes) used for compressing and storing a stripe in memory. This is a hint to the ORC writer, "
+ + "which may choose to use a smaller buffer size based on stripe size and number of columns for efficient stripe writing and memory utilization.")
+ .required(true)
+ .addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
+ .defaultValue("10 KB")
+ .build();
+
+ public static final PropertyDescriptor COMPRESSION_TYPE = new PropertyDescriptor.Builder()
+ .name("orc-compression-type")
+ .displayName("Compression Type")
+ .required(true)
+ .allowableValues("NONE", "ZLIB", "SNAPPY", "LZO")
+ .defaultValue("NONE")
+ .build();
+
+ public static final PropertyDescriptor HIVE_TABLE_NAME = new PropertyDescriptor.Builder()
+ .name("orc-hive-table-name")
+ .displayName("Hive Table Name")
+ .description("An optional table name to insert into the hive.ddl attribute. The generated DDL can be used by "
+ + "a PutHiveQL processor (presumably after a PutHDFS processor) to create a table backed by the converted ORC file. "
+ + "If this property is not provided, the full name (including namespace) of the incoming Avro record will be normalized "
+ + "and used as the table name.")
+ .required(false)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
+ // Relationships
+ static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("A FlowFile is routed to this relationship after it has been converted to ORC format.")
+ .build();
+ static final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("A FlowFile is routed to this relationship if it cannot be parsed as Avro or cannot be converted to ORC for any reason")
+ .build();
+
+ private final static List<PropertyDescriptor> propertyDescriptors;
+ private final static Set<Relationship> relationships;
+
+ private volatile Configuration orcConfig;
+
+ /*
+ * Will ensure that the list of property descriptors is built only once.
+ * Will also create a Set of relationships
+ */
+ static {
+ List<PropertyDescriptor> _propertyDescriptors = new ArrayList<>();
+ _propertyDescriptors.add(ORC_CONFIGURATION_RESOURCES);
+ _propertyDescriptors.add(STRIPE_SIZE);
+ _propertyDescriptors.add(BUFFER_SIZE);
+ _propertyDescriptors.add(COMPRESSION_TYPE);
+ _propertyDescriptors.add(HIVE_TABLE_NAME);
+ propertyDescriptors = Collections.unmodifiableList(_propertyDescriptors);
+
+ Set<Relationship> _relationships = new HashSet<>();
+ _relationships.add(REL_SUCCESS);
+ _relationships.add(REL_FAILURE);
+ relationships = Collections.unmodifiableSet(_relationships);
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return propertyDescriptors;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return relationships;
+ }
+
+ @OnScheduled
+ public void setup(ProcessContext context) {
+ boolean confFileProvided = context.getProperty(ORC_CONFIGURATION_RESOURCES).isSet();
+ if (confFileProvided) {
+ final String configFiles = context.getProperty(ORC_CONFIGURATION_RESOURCES).getValue();
+ orcConfig = HiveJdbcCommon.getConfigurationFromFiles(configFiles);
+ }
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return;
+ }
+
+ try {
+ long startTime = System.currentTimeMillis();
+ final long stripeSize = context.getProperty(STRIPE_SIZE).asDataSize(DataUnit.B).longValue();
+ final int bufferSize = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
+ final CompressionKind compressionType = CompressionKind.valueOf(context.getProperty(COMPRESSION_TYPE).getValue());
+ final AtomicReference<Schema> hiveAvroSchema = new AtomicReference<>(null);
+ final AtomicInteger totalRecordCount = new AtomicInteger(0);
+ final String fileName = flowFile.getAttribute(CoreAttributes.FILENAME.key());
+ flowFile = session.write(flowFile, (rawIn, rawOut) -> {
+ try (final InputStream in = new BufferedInputStream(rawIn);
+ final OutputStream out = new BufferedOutputStream(rawOut);
+ final DataFileStream<GenericRecord> reader = new DataFileStream<>(in, new GenericDatumReader<>())) {
+
+ // Create ORC schema from Avro schema
+ Schema avroSchema = reader.getSchema();
+
+ TypeInfo orcSchema = NiFiOrcUtils.getOrcField(avroSchema);
+
+ if (orcConfig == null) {
+ orcConfig = new Configuration();
+ }
+
+ OrcFlowFileWriter orcWriter = NiFiOrcUtils.createWriter(
+ out,
+ new Path(fileName),
+ orcConfig,
+ orcSchema,
+ stripeSize,
+ compressionType,
+ bufferSize);
+ try {
+
+ int recordCount = 0;
+ GenericRecord currRecord = null;
+ while (reader.hasNext()) {
+ currRecord = reader.next(currRecord);
+ List<Schema.Field> fields = currRecord.getSchema().getFields();
+ if (fields != null) {
+ Object[] row = new Object[fields.size()];
+ for (int i = 0; i < fields.size(); i++) {
+ Schema.Field field = fields.get(i);
+ Schema fieldSchema = field.schema();
+ Object o = currRecord.get(field.name());
+ try {
+ row[i] = NiFiOrcUtils.convertToORCObject(NiFiOrcUtils.getOrcField(fieldSchema), o);
+ } catch (ArrayIndexOutOfBoundsException aioobe) {
+ getLogger().error("Index out of bounds at record {} for column {}, type {}, and object {}",
+ new Object[]{recordCount, i, fieldSchema.getType().getName(), o.toString()},
+ aioobe);
+ throw new IOException(aioobe);
+ }
+ }
+ orcWriter.addRow(NiFiOrcUtils.createOrcStruct(orcSchema, row));
+ recordCount++;
+ }
+ }
+ hiveAvroSchema.set(avroSchema);
+ totalRecordCount.set(recordCount);
+ } finally {
+ // finished writing this record, close the writer (which will flush to the flow file)
+ orcWriter.close();
+ }
+ }
+ });
+
+ final String hiveTableName = context.getProperty(HIVE_TABLE_NAME).isSet()
+ ? context.getProperty(HIVE_TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue()
+ : NiFiOrcUtils.normalizeHiveTableName(hiveAvroSchema.get().getFullName());
+ String hiveDDL = NiFiOrcUtils.generateHiveDDL(hiveAvroSchema.get(), hiveTableName);
+ // Add attributes and transfer to success
+ flowFile = session.putAttribute(flowFile, RECORD_COUNT_ATTRIBUTE, Integer.toString(totalRecordCount.get()));
+ flowFile = session.putAttribute(flowFile, HIVE_DDL_ATTRIBUTE, hiveDDL);
+ StringBuilder newFilename = new StringBuilder();
+ int extensionIndex = fileName.lastIndexOf(".");
+ if (extensionIndex != -1) {
+ newFilename.append(fileName.substring(0, extensionIndex));
+ } else {
+ newFilename.append(fileName);
+ }
+ newFilename.append(".orc");
+ flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), ORC_MIME_TYPE);
+ flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), newFilename.toString());
+ session.transfer(flowFile, REL_SUCCESS);
+ session.getProvenanceReporter().modifyContent(flowFile, "Converted "+totalRecordCount.get()+" records", System.currentTimeMillis() - startTime);
+
+ } catch (final ProcessException pe) {
+ getLogger().error("Failed to convert {} from Avro to ORC due to {}; transferring to failure", new Object[]{flowFile, pe});
+ session.transfer(flowFile, REL_FAILURE);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/d9720239/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
index 55ee005..6f5c121 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+org.apache.nifi.processors.hive.ConvertAvroToORC
org.apache.nifi.processors.hive.SelectHiveQL
org.apache.nifi.processors.hive.PutHiveQL
org.apache.nifi.processors.hive.PutHiveStreaming
http://git-wip-us.apache.org/repos/asf/nifi/blob/d9720239/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestConvertAvroToORC.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestConvertAvroToORC.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestConvertAvroToORC.java
new file mode 100644
index 0000000..ad40198
--- /dev/null
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestConvertAvroToORC.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.hive;
+
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.io.DatumWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.io.orc.OrcFile;
+import org.apache.hadoop.hive.ql.io.orc.OrcStruct;
+import org.apache.hadoop.hive.ql.io.orc.Reader;
+import org.apache.hadoop.hive.ql.io.orc.RecordReader;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.apache.nifi.util.orc.TestNiFiOrcUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.FileOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Unit tests for ConvertAvroToORC processor
+ */
+public class TestConvertAvroToORC {
+
+ private ConvertAvroToORC processor;
+ private TestRunner runner;
+
+ @Before
+ public void setUp() throws Exception {
+ processor = new ConvertAvroToORC();
+ runner = TestRunners.newTestRunner(processor);
+ }
+
+ @Test
+ public void test_Setup() throws Exception {
+
+ }
+
+ @Test
+ public void test_onTrigger_primitive_record() throws Exception {
+ GenericData.Record record = TestNiFiOrcUtils.buildPrimitiveAvroRecord(10, 20L, true, 30.0f, 40, StandardCharsets.UTF_8.encode("Hello"), "World");
+
+ DatumWriter<GenericData.Record> writer = new GenericDatumWriter<>(record.getSchema());
+ DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(writer);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ fileWriter.create(record.getSchema(), out);
+ fileWriter.append(record);
+ // Put another record in
+ record = TestNiFiOrcUtils.buildPrimitiveAvroRecord(1, 2L, false, 3.0f, 4L, StandardCharsets.UTF_8.encode("I am"), "another record");
+ fileWriter.append(record);
+ // And one more
+ record = TestNiFiOrcUtils.buildPrimitiveAvroRecord(100, 200L, true, 300.0f, 400L, StandardCharsets.UTF_8.encode("Me"), "too!");
+ fileWriter.append(record);
+ fileWriter.flush();
+ fileWriter.close();
+ out.close();
+ Map<String, String> attributes = new HashMap<String, String>() {{
+ put(CoreAttributes.FILENAME.key(), "test.avro");
+ }};
+ runner.enqueue(out.toByteArray(), attributes);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ConvertAvroToORC.REL_SUCCESS, 1);
+
+ // Write the flow file out to disk, since the ORC Reader needs a path
+ MockFlowFile resultFlowFile = runner.getFlowFilesForRelationship(ConvertAvroToORC.REL_SUCCESS).get(0);
+ assertEquals("CREATE EXTERNAL TABLE IF NOT EXISTS test_record (int INT, long BIGINT, boolean BOOLEAN, float FLOAT, double DOUBLE, bytes BINARY, string STRING)"
+ + " STORED AS ORC", resultFlowFile.getAttribute(ConvertAvroToORC.HIVE_DDL_ATTRIBUTE));
+ assertEquals("3", resultFlowFile.getAttribute(ConvertAvroToORC.RECORD_COUNT_ATTRIBUTE));
+ assertEquals("test.orc", resultFlowFile.getAttribute(CoreAttributes.FILENAME.key()));
+ byte[] resultContents = runner.getContentAsByteArray(resultFlowFile);
+ FileOutputStream fos = new FileOutputStream("target/test1.orc");
+ fos.write(resultContents);
+ fos.flush();
+ fos.close();
+
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ Reader reader = OrcFile.createReader(new Path("target/test1.orc"), OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
+ Object o = rows.next(null);
+ assertNotNull(o);
+ assertTrue(o instanceof OrcStruct);
+ TypeInfo resultSchema = TestNiFiOrcUtils.buildPrimitiveOrcSchema();
+ StructObjectInspector inspector = (StructObjectInspector) OrcStruct.createObjectInspector(resultSchema);
+
+ // Check some fields in the first row
+ Object intFieldObject = inspector.getStructFieldData(o, inspector.getStructFieldRef("int"));
+ assertTrue(intFieldObject instanceof IntWritable);
+ assertEquals(10, ((IntWritable) intFieldObject).get());
+ Object stringFieldObject = inspector.getStructFieldData(o, inspector.getStructFieldRef("string"));
+ assertTrue(stringFieldObject instanceof Text);
+ assertEquals("World", stringFieldObject.toString());
+
+ }
+
+ @Test
+ public void test_onTrigger_complex_record() throws Exception {
+
+ Map<String, Double> mapData1 = new TreeMap<String, Double>() {{
+ put("key1", 1.0);
+ put("key2", 2.0);
+ }};
+
+ GenericData.Record record = TestNiFiOrcUtils.buildComplexAvroRecord(10, mapData1, "DEF", 3.0f, Arrays.asList(10, 20));
+
+ DatumWriter<GenericData.Record> writer = new GenericDatumWriter<>(record.getSchema());
+ DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(writer);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ fileWriter.create(record.getSchema(), out);
+ fileWriter.append(record);
+
+ // Put another record in
+ Map<String, Double> mapData2 = new TreeMap<String, Double>() {{
+ put("key1", 3.0);
+ put("key2", 4.0);
+ }};
+
+ record = TestNiFiOrcUtils.buildComplexAvroRecord(null, mapData2, "XYZ", 4L, Arrays.asList(100, 200));
+ fileWriter.append(record);
+
+ fileWriter.flush();
+ fileWriter.close();
+ out.close();
+
+ Map<String, String> attributes = new HashMap<String, String>() {{
+ put(CoreAttributes.FILENAME.key(), "test");
+ }};
+ runner.enqueue(out.toByteArray(), attributes);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ConvertAvroToORC.REL_SUCCESS, 1);
+
+ // Write the flow file out to disk, since the ORC Reader needs a path
+ MockFlowFile resultFlowFile = runner.getFlowFilesForRelationship(ConvertAvroToORC.REL_SUCCESS).get(0);
+ assertEquals("CREATE EXTERNAL TABLE IF NOT EXISTS complex_record " +
+ "(myInt INT, myMap MAP<STRING, DOUBLE>, myEnum STRING, myLongOrFloat UNIONTYPE<BIGINT, FLOAT>, myIntList ARRAY<INT>)"
+ + " STORED AS ORC", resultFlowFile.getAttribute(ConvertAvroToORC.HIVE_DDL_ATTRIBUTE));
+ assertEquals("2", resultFlowFile.getAttribute(ConvertAvroToORC.RECORD_COUNT_ATTRIBUTE));
+ assertEquals("test.orc", resultFlowFile.getAttribute(CoreAttributes.FILENAME.key()));
+ byte[] resultContents = runner.getContentAsByteArray(resultFlowFile);
+ FileOutputStream fos = new FileOutputStream("target/test1.orc");
+ fos.write(resultContents);
+ fos.flush();
+ fos.close();
+
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ Reader reader = OrcFile.createReader(new Path("target/test1.orc"), OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
+ Object o = rows.next(null);
+ assertNotNull(o);
+ assertTrue(o instanceof OrcStruct);
+ TypeInfo resultSchema = TestNiFiOrcUtils.buildComplexOrcSchema();
+ StructObjectInspector inspector = (StructObjectInspector) OrcStruct.createObjectInspector(resultSchema);
+
+ // Check some fields in the first row
+ Object intFieldObject = inspector.getStructFieldData(o, inspector.getStructFieldRef("myInt"));
+ assertTrue(intFieldObject instanceof IntWritable);
+ assertEquals(10, ((IntWritable) intFieldObject).get());
+
+ // This is pretty awkward and messy. The map object is a Map (not a MapWritable) but the keys are writables (in this case Text)
+ // and so are the values (DoubleWritables in this case).
+ Object mapFieldObject = inspector.getStructFieldData(o, inspector.getStructFieldRef("myMap"));
+ assertTrue(mapFieldObject instanceof Map);
+ Map map = (Map) mapFieldObject;
+ Object mapValue = map.get(new Text("key1"));
+ assertNotNull(mapValue);
+ assertTrue(mapValue instanceof DoubleWritable);
+ assertEquals(1.0, ((DoubleWritable) mapValue).get(), Double.MIN_VALUE);
+
+ mapValue = map.get(new Text("key2"));
+ assertNotNull(mapValue);
+ assertTrue(mapValue instanceof DoubleWritable);
+ assertEquals(2.0, ((DoubleWritable) mapValue).get(), Double.MIN_VALUE);
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/d9720239/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHiveStreaming.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHiveStreaming.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHiveStreaming.java
index 3876e74..f95a377 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHiveStreaming.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHiveStreaming.java
@@ -684,12 +684,12 @@ public class TestPutHiveStreaming {
@Override
protected void closeConnection() throws InterruptedException {
- // TODO
+ // Empty
}
@Override
protected void commitTxn() throws CommitFailure, InterruptedException {
- // TODO
+ // Empty
}
@Override
@@ -700,17 +700,17 @@ public class TestPutHiveStreaming {
@Override
protected void closeTxnBatch() throws InterruptedException {
- // TODO
+ // Empty
}
@Override
protected void abortTxn() throws InterruptedException {
- // TODO
+ // Empty
}
@Override
protected void nextTxn(boolean rollToNext) throws StreamingException, InterruptedException, TxnBatchFailure {
- // TODO
+ // Empty
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/d9720239/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java
new file mode 100644
index 0000000..556723c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.orc;
+
+
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.generic.GenericData;
+import org.apache.hadoop.hive.ql.io.orc.NiFiOrcUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit tests for the NiFiOrcUtils helper class
+ */
+public class TestNiFiOrcUtils {
+
+ @Test
+ public void test_getOrcField_primitive() throws Exception {
+ // Expected ORC types
+ TypeInfo[] expectedTypes = {
+ TypeInfoFactory.getPrimitiveTypeInfo("int"),
+ TypeInfoFactory.getPrimitiveTypeInfo("bigint"),
+ TypeInfoFactory.getPrimitiveTypeInfo("boolean"),
+ TypeInfoFactory.getPrimitiveTypeInfo("float"),
+ TypeInfoFactory.getPrimitiveTypeInfo("double"),
+ TypeInfoFactory.getPrimitiveTypeInfo("binary"),
+ TypeInfoFactory.getPrimitiveTypeInfo("string")
+ };
+
+ // Build a fake Avro record with all types
+ Schema testSchema = buildPrimitiveAvroSchema();
+ List<Schema.Field> fields = testSchema.getFields();
+ for (int i = 0; i < fields.size(); i++) {
+ assertEquals(expectedTypes[i], NiFiOrcUtils.getOrcField(fields.get(i).schema()));
+ }
+
+ }
+
+ @Test
+ public void test_getOrcField_union_optional_type() throws Exception {
+ final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
+ builder.name("union").type().unionOf().nullBuilder().endNull().and().booleanType().endUnion().noDefault();
+ Schema testSchema = builder.endRecord();
+ TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema.getField("union").schema());
+ assertEquals(TypeInfoCreator.createBoolean(), orcType);
+ }
+
+ @Test
+ public void test_getOrcField_union() throws Exception {
+ final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
+ builder.name("union").type().unionOf().intType().and().booleanType().endUnion().noDefault();
+ Schema testSchema = builder.endRecord();
+ TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema.getField("union").schema());
+ assertEquals(
+ TypeInfoFactory.getUnionTypeInfo(Arrays.asList(
+ TypeInfoCreator.createInt(),
+ TypeInfoCreator.createBoolean())),
+ orcType);
+ }
+
+ @Test
+ public void test_getOrcField_map() throws Exception {
+ final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
+ builder.name("map").type().map().values().doubleType().noDefault();
+ Schema testSchema = builder.endRecord();
+ TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema.getField("map").schema());
+ assertEquals(
+ TypeInfoFactory.getMapTypeInfo(
+ TypeInfoCreator.createString(),
+ TypeInfoCreator.createDouble()),
+ orcType);
+ }
+
+ @Test
+ public void test_getOrcField_nested_map() throws Exception {
+ final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
+ builder.name("map").type().map().values().map().values().doubleType().noDefault();
+ Schema testSchema = builder.endRecord();
+ TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema.getField("map").schema());
+ assertEquals(
+ TypeInfoFactory.getMapTypeInfo(TypeInfoCreator.createString(),
+ TypeInfoFactory.getMapTypeInfo(TypeInfoCreator.createString(), TypeInfoCreator.createDouble())),
+ orcType);
+ }
+
+ @Test
+ public void test_getOrcField_array() throws Exception {
+ final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
+ builder.name("array").type().array().items().longType().noDefault();
+ Schema testSchema = builder.endRecord();
+ TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema.getField("array").schema());
+ assertEquals(
+ TypeInfoFactory.getListTypeInfo(TypeInfoCreator.createLong()),
+ orcType);
+ }
+
+ @Test
+ public void test_getOrcField_complex_array() throws Exception {
+ final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
+ builder.name("array").type().array().items().map().values().floatType().noDefault();
+ Schema testSchema = builder.endRecord();
+ TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema.getField("array").schema());
+ assertEquals(
+ TypeInfoFactory.getListTypeInfo(TypeInfoFactory.getMapTypeInfo(TypeInfoCreator.createString(), TypeInfoCreator.createFloat())),
+ orcType);
+ }
+
+ @Test
+ public void test_getOrcField_record() throws Exception {
+ final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
+ builder.name("int").type().intType().noDefault();
+ builder.name("long").type().longType().longDefault(1L);
+ builder.name("array").type().array().items().stringType().noDefault();
+ Schema testSchema = builder.endRecord();
+ TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema);
+ assertEquals(
+ TypeInfoFactory.getStructTypeInfo(
+ Arrays.asList("int", "long", "array"),
+ Arrays.asList(
+ TypeInfoCreator.createInt(),
+ TypeInfoCreator.createLong(),
+ TypeInfoFactory.getListTypeInfo(TypeInfoCreator.createString()))),
+ orcType);
+ }
+
+ @Test
+ public void test_getOrcField_enum() throws Exception {
+ final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
+ builder.name("enumField").type().enumeration("enum").symbols("a", "b", "c").enumDefault("a");
+ Schema testSchema = builder.endRecord();
+ TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema.getField("enumField").schema());
+ assertEquals(TypeInfoCreator.createString(), orcType);
+ }
+
+ @Test
+ public void test_getPrimitiveOrcTypeFromPrimitiveAvroType() throws Exception {
+ // Expected ORC types
+ TypeInfo[] expectedTypes = {
+ TypeInfoCreator.createInt(),
+ TypeInfoCreator.createLong(),
+ TypeInfoCreator.createBoolean(),
+ TypeInfoCreator.createFloat(),
+ TypeInfoCreator.createDouble(),
+ TypeInfoCreator.createBinary(),
+ TypeInfoCreator.createString(),
+ };
+
+ Schema testSchema = buildPrimitiveAvroSchema();
+ List<Schema.Field> fields = testSchema.getFields();
+ for (int i = 0; i < fields.size(); i++) {
+ assertEquals(expectedTypes[i], NiFiOrcUtils.getPrimitiveOrcTypeFromPrimitiveAvroType(fields.get(i).schema().getType()));
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void test_getPrimitiveOrcTypeFromPrimitiveAvroType_badType() throws Exception {
+ Schema.Type nonPrimitiveType = Schema.Type.ARRAY;
+ NiFiOrcUtils.getPrimitiveOrcTypeFromPrimitiveAvroType(nonPrimitiveType);
+ }
+
+ @Test
+ public void test_getWritable() throws Exception {
+ assertTrue(NiFiOrcUtils.convertToORCObject(null, 1) instanceof IntWritable);
+ assertTrue(NiFiOrcUtils.convertToORCObject(null, 1L) instanceof LongWritable);
+ assertTrue(NiFiOrcUtils.convertToORCObject(null, 1.0f) instanceof FloatWritable);
+ assertTrue(NiFiOrcUtils.convertToORCObject(null, 1.0) instanceof DoubleWritable);
+ assertTrue(NiFiOrcUtils.convertToORCObject(null, new int[]{1, 2, 3}) instanceof List);
+ assertTrue(NiFiOrcUtils.convertToORCObject(null, Arrays.asList(1, 2, 3)) instanceof List);
+ Map<String, Float> map = new HashMap<>();
+ map.put("Hello", 1.0f);
+ map.put("World", 2.0f);
+
+ Object writable = NiFiOrcUtils.convertToORCObject(TypeInfoUtils.getTypeInfoFromTypeString("map<string,float>"), map);
+ assertTrue(writable instanceof MapWritable);
+ MapWritable mapWritable = (MapWritable) writable;
+ mapWritable.forEach((key, value) -> {
+ assertTrue(key instanceof Text);
+ assertTrue(value instanceof FloatWritable);
+ });
+ }
+
+ @Test
+ public void test_getHiveTypeFromAvroType_primitive() throws Exception {
+ // Expected ORC types
+ String[] expectedTypes = {
+ "INT",
+ "BIGINT",
+ "BOOLEAN",
+ "FLOAT",
+ "DOUBLE",
+ "BINARY",
+ "STRING",
+ };
+
+ Schema testSchema = buildPrimitiveAvroSchema();
+ List<Schema.Field> fields = testSchema.getFields();
+ for (int i = 0; i < fields.size(); i++) {
+ assertEquals(expectedTypes[i], NiFiOrcUtils.getHiveTypeFromAvroType(fields.get(i).schema()));
+ }
+ }
+
+ @Test
+ public void test_getHiveTypeFromAvroType_complex() throws Exception {
+ // Expected ORC types
+ String[] expectedTypes = {
+ "INT",
+ "MAP<STRING, DOUBLE>",
+ "STRING",
+ "UNIONTYPE<BIGINT, FLOAT>",
+ "ARRAY<INT>"
+ };
+
+ Schema testSchema = buildComplexAvroSchema();
+ List<Schema.Field> fields = testSchema.getFields();
+ for (int i = 0; i < fields.size(); i++) {
+ assertEquals(expectedTypes[i], NiFiOrcUtils.getHiveTypeFromAvroType(fields.get(i).schema()));
+ }
+
+ assertEquals("STRUCT<myInt:INT, myMap:MAP<STRING, DOUBLE>, myEnum:STRING, myLongOrFloat:UNIONTYPE<BIGINT, FLOAT>, myIntList:ARRAY<INT>>",
+ NiFiOrcUtils.getHiveTypeFromAvroType(testSchema));
+ }
+
+ @Test
+ public void test_generateHiveDDL_primitive() throws Exception {
+ Schema avroSchema = buildPrimitiveAvroSchema();
+ String ddl = NiFiOrcUtils.generateHiveDDL(avroSchema, "myHiveTable");
+ assertEquals("CREATE EXTERNAL TABLE IF NOT EXISTS myHiveTable (int INT, long BIGINT, boolean BOOLEAN, float FLOAT, double DOUBLE, bytes BINARY, string STRING)"
+ + " STORED AS ORC", ddl);
+ }
+
+ @Test
+ public void test_generateHiveDDL_complex() throws Exception {
+ Schema avroSchema = buildComplexAvroSchema();
+ String ddl = NiFiOrcUtils.generateHiveDDL(avroSchema, "myHiveTable");
+ assertEquals("CREATE EXTERNAL TABLE IF NOT EXISTS myHiveTable "
+ + "(myInt INT, myMap MAP<STRING, DOUBLE>, myEnum STRING, myLongOrFloat UNIONTYPE<BIGINT, FLOAT>, myIntList ARRAY<INT>)"
+ + " STORED AS ORC", ddl);
+ }
+
+
+ //////////////////
+ // Helper methods
+ //////////////////
+
+ public static Schema buildPrimitiveAvroSchema() {
+ // Build a fake Avro record with all primitive types
+ final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("test.record").namespace("any.data").fields();
+ builder.name("int").type().intType().noDefault();
+ builder.name("long").type().longType().longDefault(1L);
+ builder.name("boolean").type().booleanType().booleanDefault(true);
+ builder.name("float").type().floatType().floatDefault(0.0f);
+ builder.name("double").type().doubleType().doubleDefault(0.0);
+ builder.name("bytes").type().bytesType().noDefault();
+ builder.name("string").type().stringType().stringDefault("default");
+ return builder.endRecord();
+ }
+
+ public static GenericData.Record buildPrimitiveAvroRecord(int i, long l, boolean b, float f, double d, ByteBuffer bytes, String string) {
+ Schema schema = buildPrimitiveAvroSchema();
+ GenericData.Record row = new GenericData.Record(schema);
+ row.put("int", i);
+ row.put("long", l);
+ row.put("boolean", b);
+ row.put("float", f);
+ row.put("double", d);
+ row.put("bytes", bytes);
+ row.put("string", string);
+ return row;
+ }
+
+ public static TypeInfo buildPrimitiveOrcSchema() {
+ return TypeInfoFactory.getStructTypeInfo(Arrays.asList("int", "long", "boolean", "float", "double", "bytes", "string"),
+ Arrays.asList(
+ TypeInfoCreator.createInt(),
+ TypeInfoCreator.createLong(),
+ TypeInfoCreator.createBoolean(),
+ TypeInfoCreator.createFloat(),
+ TypeInfoCreator.createDouble(),
+ TypeInfoCreator.createBinary(),
+ TypeInfoCreator.createString()));
+ }
+
+ public static Schema buildComplexAvroSchema() {
+ // Build a fake Avro record with nested types
+ final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("complex.record").namespace("any.data").fields();
+ builder.name("myInt").type().unionOf().nullType().and().intType().endUnion().nullDefault();
+ builder.name("myMap").type().map().values().doubleType().noDefault();
+ builder.name("myEnum").type().enumeration("myEnum").symbols("ABC", "DEF", "XYZ").enumDefault("ABC");
+ builder.name("myLongOrFloat").type().unionOf().longType().and().floatType().endUnion().noDefault();
+ builder.name("myIntList").type().array().items().intType().noDefault();
+ return builder.endRecord();
+ }
+
+ public static GenericData.Record buildComplexAvroRecord(Integer i, Map<String, Double> m, String e, Object unionVal, List<Integer> intArray) {
+ Schema schema = buildComplexAvroSchema();
+ GenericData.Record row = new GenericData.Record(schema);
+ row.put("myInt", i);
+ row.put("myMap", m);
+ row.put("myEnum", e);
+ row.put("myLongOrFloat", unionVal);
+ row.put("myIntList", intArray);
+ return row;
+ }
+
+ public static TypeInfo buildComplexOrcSchema() {
+ return TypeInfoUtils.getTypeInfoFromTypeString("struct<myInt:int,myMap:map<string,double>,myEnum:string,myLongOrFloat:uniontype<int>,myIntList:array<int>>");
+ }
+
+
+ private static class TypeInfoCreator {
+ static TypeInfo createInt() {
+ return TypeInfoFactory.getPrimitiveTypeInfo("int");
+ }
+
+ static TypeInfo createLong() {
+ return TypeInfoFactory.getPrimitiveTypeInfo("bigint");
+ }
+
+ static TypeInfo createBoolean() {
+ return TypeInfoFactory.getPrimitiveTypeInfo("boolean");
+ }
+
+ static TypeInfo createFloat() {
+ return TypeInfoFactory.getPrimitiveTypeInfo("float");
+ }
+
+ static TypeInfo createDouble() {
+ return TypeInfoFactory.getPrimitiveTypeInfo("double");
+ }
+
+ static TypeInfo createBinary() {
+ return TypeInfoFactory.getPrimitiveTypeInfo("binary");
+ }
+
+ static TypeInfo createString() {
+ return TypeInfoFactory.getPrimitiveTypeInfo("string");
+ }
+ }
+}
\ No newline at end of file
[2/3] nifi git commit: NIFI-1663: Add ConvertAvroToORC processor
Posted by oz...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/d9720239/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/OrcFlowFileWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/OrcFlowFileWriter.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/OrcFlowFileWriter.java
new file mode 100644
index 0000000..a98092a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/OrcFlowFileWriter.java
@@ -0,0 +1,2650 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.management.ManagementFactory;
+import java.nio.ByteBuffer;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.io.IOConstants;
+import org.apache.hadoop.hive.ql.io.filters.BloomFilterIO;
+import org.apache.hadoop.hive.ql.io.orc.CompressionCodec.Modifier;
+import org.apache.hadoop.hive.ql.io.orc.OrcFile.CompressionStrategy;
+import org.apache.hadoop.hive.ql.io.orc.OrcFile.EncodingStrategy;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Longs;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
+import org.apache.nifi.stream.io.ByteCountingOutputStream;
+
+/**
+ * An ORC file writer. The file is divided into stripes, which is the natural
+ * unit of work when reading. Each stripe is buffered in memory until the
+ * memory reaches the stripe size and then it is written out broken down by
+ * columns. Each column is written by a TreeWriter that is specific to that
+ * type of column. TreeWriters may have children TreeWriters that handle the
+ * sub-types. Each of the TreeWriters writes the column's data as a set of
+ * streams.
+ * <p>
+ * This class is synchronized so that multi-threaded access is ok. In
+ * particular, because the MemoryManager is shared between writers, this class
+ * assumes that checkMemory may be called from a separate thread.
+ */
+public class OrcFlowFileWriter implements Writer, MemoryManager.Callback {
+
+ private static final Log LOG = LogFactory.getLog(WriterImpl.class);
+
+ private static final int HDFS_BUFFER_SIZE = 256 * 1024;
+ private static final int MIN_ROW_INDEX_STRIDE = 1000;
+
+ // threshold above which buffer size will be automatically resized
+ private static final int COLUMN_COUNT_THRESHOLD = 1000;
+
+ private final Path path;
+ private final long defaultStripeSize;
+ private long adjustedStripeSize;
+ private final int rowIndexStride;
+ private final CompressionKind compress;
+ private final CompressionCodec codec;
+ private final boolean addBlockPadding;
+ private final int bufferSize;
+ private final long blockSize;
+ private final float paddingTolerance;
+ // the streams that make up the current stripe
+ private final Map<StreamName, BufferedStream> streams = new TreeMap<>();
+
+ private final OutputStream flowFileOutputStream;
+
+ private ByteCountingOutputStream rawWriter = null;
+ // the compressed metadata information outStream
+ private OutStream writer = null;
+ // a protobuf outStream around streamFactory
+ private CodedOutputStream protobufWriter = null;
+ private long headerLength;
+ private int columnCount;
+ private long rowCount = 0;
+ private long rowsInStripe = 0;
+ private long rawDataSize = 0;
+ private int rowsInIndex = 0;
+ private int stripesAtLastFlush = -1;
+ private final List<OrcProto.StripeInformation> stripes = new ArrayList<>();
+ private final Map<String, ByteString> userMetadata = new TreeMap<>();
+ private final StreamFactory streamFactory = new StreamFactory();
+ private final TreeWriter treeWriter;
+ private final boolean buildIndex;
+ private final MemoryManager memoryManager;
+ private final OrcFile.Version version;
+ private final Configuration conf;
+ private final OrcFile.WriterCallback callback;
+ private final OrcFile.WriterContext callbackContext;
+ private final OrcFile.EncodingStrategy encodingStrategy;
+ private final OrcFile.CompressionStrategy compressionStrategy;
+ private final boolean[] bloomFilterColumns;
+ private final double bloomFilterFpp;
+
+
+ public OrcFlowFileWriter(OutputStream flowFileOutputStream,
+ Path path,
+ Configuration conf,
+ ObjectInspector inspector,
+ long stripeSize,
+ CompressionKind compress,
+ int bufferSize,
+ int rowIndexStride,
+ MemoryManager memoryManager,
+ boolean addBlockPadding,
+ OrcFile.Version version,
+ OrcFile.WriterCallback callback,
+ EncodingStrategy encodingStrategy,
+ CompressionStrategy compressionStrategy,
+ float paddingTolerance,
+ long blockSizeValue,
+ String bloomFilterColumnNames,
+ double bloomFilterFpp) throws IOException {
+ this.flowFileOutputStream = flowFileOutputStream;
+ this.path = path;
+ this.conf = conf;
+ this.callback = callback;
+ callbackContext = (callback != null) ? () -> OrcFlowFileWriter.this : null;
+ this.adjustedStripeSize = stripeSize;
+ this.defaultStripeSize = stripeSize;
+ this.version = version;
+ this.encodingStrategy = encodingStrategy;
+ this.compressionStrategy = compressionStrategy;
+ this.addBlockPadding = addBlockPadding;
+ this.blockSize = blockSizeValue;
+ this.paddingTolerance = paddingTolerance;
+ this.compress = compress;
+ this.rowIndexStride = rowIndexStride;
+ this.memoryManager = memoryManager;
+ buildIndex = rowIndexStride > 0;
+ codec = createCodec(compress);
+ String allColumns = conf.get(IOConstants.COLUMNS);
+ if (allColumns == null) {
+ allColumns = getColumnNamesFromInspector(inspector);
+ }
+ this.bufferSize = getEstimatedBufferSize(allColumns, bufferSize);
+ if (version == OrcFile.Version.V_0_11) {
+ /* do not write bloom filters for ORC v11 */
+ this.bloomFilterColumns =
+ OrcUtils.includeColumns(null, allColumns, inspector);
+ } else {
+ this.bloomFilterColumns =
+ OrcUtils.includeColumns(bloomFilterColumnNames, allColumns, inspector);
+ }
+ this.bloomFilterFpp = bloomFilterFpp;
+ treeWriter = createTreeWriter(inspector, streamFactory, false);
+ if (buildIndex && rowIndexStride < MIN_ROW_INDEX_STRIDE) {
+ throw new IllegalArgumentException("Row stride must be at least " +
+ MIN_ROW_INDEX_STRIDE);
+ }
+
+ // ensure that we are able to handle callbacks before we register ourselves
+ memoryManager.addWriter(path, stripeSize, this);
+ }
+
+ private String getColumnNamesFromInspector(ObjectInspector inspector) {
+ List<String> fieldNames = Lists.newArrayList();
+ Joiner joiner = Joiner.on(",");
+ if (inspector instanceof StructObjectInspector) {
+ StructObjectInspector soi = (StructObjectInspector) inspector;
+ List<? extends StructField> fields = soi.getAllStructFieldRefs();
+ fieldNames.addAll(fields.stream().map((Function<StructField, String>) StructField::getFieldName).collect(Collectors.toList()));
+ }
+ return joiner.join(fieldNames);
+ }
+
+ @VisibleForTesting
+ int getEstimatedBufferSize(int bs) {
+ return getEstimatedBufferSize(conf.get(IOConstants.COLUMNS), bs);
+ }
+
+ int getEstimatedBufferSize(String colNames, int bs) {
+ long availableMem = getMemoryAvailableForORC();
+ if (colNames != null) {
+ final int numCols = colNames.split(",").length;
+ if (numCols > COLUMN_COUNT_THRESHOLD) {
+ // In BufferedStream, there are 3 outstream buffers (compressed,
+ // uncompressed and overflow) and list of previously compressed buffers.
+ // Since overflow buffer is rarely used, lets consider only 2 allocation.
+ // Also, initially, the list of compression buffers will be empty.
+ final int outStreamBuffers = codec == null ? 1 : 2;
+
+ // max possible streams per column is 5. For string columns, there is
+ // ROW_INDEX, PRESENT, DATA, LENGTH, DICTIONARY_DATA streams.
+ final int maxStreams = 5;
+
+ // Lets assume 10% memory for holding dictionary in memory and other
+ // object allocations
+ final long miscAllocation = (long) (0.1f * availableMem);
+
+ // compute the available memory
+ final long remainingMem = availableMem - miscAllocation;
+
+ int estBufferSize = (int) (remainingMem / (maxStreams * outStreamBuffers * numCols));
+ estBufferSize = getClosestBufferSize(estBufferSize, bs);
+ if (estBufferSize > bs) {
+ estBufferSize = bs;
+ }
+
+ LOG.info("WIDE TABLE - Number of columns: " + numCols + " Chosen compression buffer size: " + estBufferSize);
+ return estBufferSize;
+ }
+ }
+ return bs;
+ }
+
+ private int getClosestBufferSize(int estBufferSize, int bs) {
+ final int kb4 = 4 * 1024;
+ final int kb8 = 8 * 1024;
+ final int kb16 = 16 * 1024;
+ final int kb32 = 32 * 1024;
+ final int kb64 = 64 * 1024;
+ final int kb128 = 128 * 1024;
+ final int kb256 = 256 * 1024;
+ if (estBufferSize <= kb4) {
+ return kb4;
+ } else if (estBufferSize > kb4 && estBufferSize <= kb8) {
+ return kb8;
+ } else if (estBufferSize > kb8 && estBufferSize <= kb16) {
+ return kb16;
+ } else if (estBufferSize > kb16 && estBufferSize <= kb32) {
+ return kb32;
+ } else if (estBufferSize > kb32 && estBufferSize <= kb64) {
+ return kb64;
+ } else if (estBufferSize > kb64 && estBufferSize <= kb128) {
+ return kb128;
+ } else {
+ return kb256;
+ }
+ }
+
+ // the assumption is only one ORC writer open at a time, which holds true for
+ // most of the cases. HIVE-6455 forces single writer case.
+ private long getMemoryAvailableForORC() {
+ HiveConf.ConfVars poolVar = HiveConf.ConfVars.HIVE_ORC_FILE_MEMORY_POOL;
+ double maxLoad = conf.getFloat(poolVar.varname, poolVar.defaultFloatVal);
+ long totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax() * maxLoad);
+ return totalMemoryPool;
+ }
+
+ public static CompressionCodec createCodec(CompressionKind kind) {
+ switch (kind) {
+ case NONE:
+ return null;
+ case ZLIB:
+ return new ZlibCodec();
+ case SNAPPY:
+ return new SnappyCodec();
+ case LZO:
+ try {
+ Class<? extends CompressionCodec> lzo =
+ (Class<? extends CompressionCodec>)
+ JavaUtils.loadClass("org.apache.hadoop.hive.ql.io.orc.LzoCodec");
+ return lzo.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException("LZO is not available.", e);
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException("Problem initializing LZO", e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException("Insufficient access to LZO", e);
+ }
+ default:
+ throw new IllegalArgumentException("Unknown compression codec: " +
+ kind);
+ }
+ }
+
+ @Override
+ public synchronized boolean checkMemory(double newScale) throws IOException {
+ long limit = Math.round(adjustedStripeSize * newScale);
+ long size = estimateStripeSize();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ORC writer " + path + " size = " + size + " limit = " +
+ limit);
+ }
+ if (size > limit) {
+ flushStripe();
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * This class is used to hold the contents of streams as they are buffered.
+ * The TreeWriters write to the outStream and the codec compresses the
+ * data as buffers fill up and stores them in the output list. When the
+ * stripe is being written, the whole stream is written to the file.
+ */
+ private class BufferedStream implements OutStream.OutputReceiver {
+ private final OutStream outStream;
+ private final List<ByteBuffer> output = new ArrayList<ByteBuffer>();
+
+ BufferedStream(String name, int bufferSize,
+ CompressionCodec codec) throws IOException {
+ outStream = new OutStream(name, bufferSize, codec, this);
+ }
+
+ /**
+ * Receive a buffer from the compression codec.
+ *
+ * @param buffer the buffer to save
+ * @throws IOException if an error occurs while receiving a buffer
+ */
+ @Override
+ public void output(ByteBuffer buffer) {
+ output.add(buffer);
+ }
+
+ /**
+ * Get the number of bytes in buffers that are allocated to this stream.
+ *
+ * @return number of bytes in buffers
+ */
+ public long getBufferSize() {
+ long result = 0;
+ for (ByteBuffer buf : output) {
+ result += buf.capacity();
+ }
+ return outStream.getBufferSize() + result;
+ }
+
+ /**
+ * Flush the stream to the codec.
+ *
+ * @throws IOException if an error occurs while flushing the stream
+ */
+ public void flush() throws IOException {
+ outStream.flush();
+ }
+
+ /**
+ * Clear all of the buffers.
+ *
+ * @throws IOException if an error occurs while clearing the buffers
+ */
+ public void clear() throws IOException {
+ outStream.clear();
+ output.clear();
+ }
+
+ /**
+ * Check the state of suppress flag in output stream
+ *
+ * @return value of suppress flag
+ */
+ public boolean isSuppressed() {
+ return outStream.isSuppressed();
+ }
+
+ /**
+ * Get the number of bytes that will be written to the output. Assumes
+ * the stream has already been flushed.
+ *
+ * @return the number of bytes
+ */
+ public long getOutputSize() {
+ long result = 0;
+ for (ByteBuffer buffer : output) {
+ result += buffer.remaining();
+ }
+ return result;
+ }
+
+ /**
+ * Write the saved compressed buffers to the OutputStream.
+ *
+ * @param out the stream to write to
+ * @throws IOException if an error occurs during write
+ */
+ void spillTo(OutputStream out) throws IOException {
+ for (ByteBuffer buffer : output) {
+ out.write(buffer.array(), buffer.arrayOffset() + buffer.position(),
+ buffer.remaining());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return outStream.toString();
+ }
+ }
+
+ /**
+ * An output receiver that writes the ByteBuffers to the output stream
+ * as they are received.
+ */
+ private class DirectStream implements OutStream.OutputReceiver {
+ private final OutputStream output;
+
+ DirectStream(OutputStream output) {
+ this.output = output;
+ }
+
+ @Override
+ public void output(ByteBuffer buffer) throws IOException {
+ output.write(buffer.array(), buffer.arrayOffset() + buffer.position(),
+ buffer.remaining());
+ }
+ }
+
+ private static class RowIndexPositionRecorder implements PositionRecorder {
+ private final OrcProto.RowIndexEntry.Builder builder;
+
+ RowIndexPositionRecorder(OrcProto.RowIndexEntry.Builder builder) {
+ this.builder = builder;
+ }
+
+ @Override
+ public void addPosition(long position) {
+ builder.addPositions(position);
+ }
+ }
+
+ /**
+ * Interface from the Writer to the TreeWriters. This limits the visibility
+ * that the TreeWriters have into the Writer.
+ */
+ private class StreamFactory {
+ /**
+ * Create a stream to store part of a column.
+ *
+ * @param column the column id for the stream
+ * @param kind the kind of stream
+ * @return The output outStream that the section needs to be written to.
+ * @throws IOException if an error occurs while creating the stream
+ */
+ public OutStream createStream(int column,
+ OrcProto.Stream.Kind kind
+ ) throws IOException {
+ final StreamName name = new StreamName(column, kind);
+ final EnumSet<CompressionCodec.Modifier> modifiers;
+
+ switch (kind) {
+ case BLOOM_FILTER:
+ case DATA:
+ case DICTIONARY_DATA:
+ if (getCompressionStrategy() == CompressionStrategy.SPEED) {
+ modifiers = EnumSet.of(Modifier.FAST, Modifier.TEXT);
+ } else {
+ modifiers = EnumSet.of(Modifier.DEFAULT, Modifier.TEXT);
+ }
+ break;
+ case LENGTH:
+ case DICTIONARY_COUNT:
+ case PRESENT:
+ case ROW_INDEX:
+ case SECONDARY:
+ // easily compressed using the fastest modes
+ modifiers = EnumSet.of(Modifier.FASTEST, Modifier.BINARY);
+ break;
+ default:
+ LOG.warn("Missing ORC compression modifiers for " + kind);
+ modifiers = null;
+ break;
+ }
+
+ BufferedStream result = streams.get(name);
+ if (result == null) {
+ result = new BufferedStream(name.toString(), bufferSize,
+ codec == null ? codec : codec.modify(modifiers));
+ streams.put(name, result);
+ }
+ return result.outStream;
+ }
+
+ /**
+ * Get the next column id.
+ *
+ * @return a number from 0 to the number of columns - 1
+ */
+ public int getNextColumnId() {
+ return columnCount++;
+ }
+
+ /**
+ * Get the current column id. After creating all tree writers this count should tell how many
+ * columns (including columns within nested complex objects) are created in total.
+ *
+ * @return current column id
+ */
+ public int getCurrentColumnId() {
+ return columnCount;
+ }
+
+ /**
+ * Get the stride rate of the row index.
+ */
+ public int getRowIndexStride() {
+ return rowIndexStride;
+ }
+
+ /**
+ * Should be building the row index.
+ *
+ * @return true if we are building the index
+ */
+ public boolean buildIndex() {
+ return buildIndex;
+ }
+
+ /**
+ * Is the ORC file compressed?
+ *
+ * @return are the streams compressed
+ */
+ public boolean isCompressed() {
+ return codec != null;
+ }
+
+ /**
+ * Get the encoding strategy to use.
+ *
+ * @return encoding strategy
+ */
+ public EncodingStrategy getEncodingStrategy() {
+ return encodingStrategy;
+ }
+
+ /**
+ * Get the compression strategy to use.
+ *
+ * @return compression strategy
+ */
+ public CompressionStrategy getCompressionStrategy() {
+ return compressionStrategy;
+ }
+
+ /**
+ * Get the bloom filter columns
+ *
+ * @return bloom filter columns
+ */
+ public boolean[] getBloomFilterColumns() {
+ return bloomFilterColumns;
+ }
+
+ /**
+ * Get bloom filter false positive percentage.
+ *
+ * @return fpp
+ */
+ public double getBloomFilterFPP() {
+ return bloomFilterFpp;
+ }
+
+ /**
+ * Get the writer's configuration.
+ *
+ * @return configuration
+ */
+ public Configuration getConfiguration() {
+ return conf;
+ }
+
+ /**
+ * Get the version of the file to write.
+ */
+ public OrcFile.Version getVersion() {
+ return version;
+ }
+ }
+
+ /**
+ * The parent class of all of the writers for each column. Each column
+ * is written by an instance of this class. The compound types (struct,
+ * list, map, and union) have children tree writers that write the children
+ * types.
+ */
+ private abstract static class TreeWriter {
+ protected final int id;
+ protected final ObjectInspector inspector;
+ private final BitFieldWriter isPresent;
+ private final boolean isCompressed;
+ protected final ColumnStatisticsImpl indexStatistics;
+ protected final ColumnStatisticsImpl stripeColStatistics;
+ private final ColumnStatisticsImpl fileStatistics;
+ protected TreeWriter[] childrenWriters;
+ protected final RowIndexPositionRecorder rowIndexPosition;
+ private final OrcProto.RowIndex.Builder rowIndex;
+ private final OrcProto.RowIndexEntry.Builder rowIndexEntry;
+ private final PositionedOutputStream rowIndexStream;
+ private final PositionedOutputStream bloomFilterStream;
+ protected final BloomFilterIO bloomFilter;
+ protected final boolean createBloomFilter;
+ private final OrcProto.BloomFilterIndex.Builder bloomFilterIndex;
+ private final OrcProto.BloomFilter.Builder bloomFilterEntry;
+ private boolean foundNulls;
+ private OutStream isPresentOutStream;
+ private final List<StripeStatistics.Builder> stripeStatsBuilders;
+
+ /**
+ * Create a tree writer.
+ *
+ * @param columnId the column id of the column to write
+ * @param inspector the object inspector to use
+ * @param streamFactory limited access to the Writer's data.
+ * @param nullable can the value be null?
+ * @throws IOException if an error occurs during creation
+ */
+ TreeWriter(int columnId, ObjectInspector inspector,
+ StreamFactory streamFactory,
+ boolean nullable) throws IOException {
+ this.isCompressed = streamFactory.isCompressed();
+ this.id = columnId;
+ this.inspector = inspector;
+ if (nullable) {
+ isPresentOutStream = streamFactory.createStream(id,
+ OrcProto.Stream.Kind.PRESENT);
+ isPresent = new BitFieldWriter(isPresentOutStream, 1);
+ } else {
+ isPresent = null;
+ }
+ this.foundNulls = false;
+ createBloomFilter = streamFactory.getBloomFilterColumns()[columnId];
+ indexStatistics = ColumnStatisticsImpl.create(inspector);
+ stripeColStatistics = ColumnStatisticsImpl.create(inspector);
+ fileStatistics = ColumnStatisticsImpl.create(inspector);
+ childrenWriters = new TreeWriter[0];
+ rowIndex = OrcProto.RowIndex.newBuilder();
+ rowIndexEntry = OrcProto.RowIndexEntry.newBuilder();
+ rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry);
+ stripeStatsBuilders = Lists.newArrayList();
+ if (streamFactory.buildIndex()) {
+ rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Kind.ROW_INDEX);
+ } else {
+ rowIndexStream = null;
+ }
+ if (createBloomFilter) {
+ bloomFilterEntry = OrcProto.BloomFilter.newBuilder();
+ bloomFilterIndex = OrcProto.BloomFilterIndex.newBuilder();
+ bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Kind.BLOOM_FILTER);
+ bloomFilter = new BloomFilterIO(streamFactory.getRowIndexStride(),
+ streamFactory.getBloomFilterFPP());
+ } else {
+ bloomFilterEntry = null;
+ bloomFilterIndex = null;
+ bloomFilterStream = null;
+ bloomFilter = null;
+ }
+ }
+
+ protected OrcProto.RowIndex.Builder getRowIndex() {
+ return rowIndex;
+ }
+
+ protected ColumnStatisticsImpl getStripeStatistics() {
+ return stripeColStatistics;
+ }
+
+ protected ColumnStatisticsImpl getFileStatistics() {
+ return fileStatistics;
+ }
+
+ protected OrcProto.RowIndexEntry.Builder getRowIndexEntry() {
+ return rowIndexEntry;
+ }
+
+ IntegerWriter createIntegerWriter(PositionedOutputStream output,
+ boolean signed, boolean isDirectV2,
+ StreamFactory writer) {
+ if (isDirectV2) {
+ boolean alignedBitpacking = false;
+ if (writer.getEncodingStrategy().equals(EncodingStrategy.SPEED)) {
+ alignedBitpacking = true;
+ }
+ return new RunLengthIntegerWriterV2(output, signed, alignedBitpacking);
+ } else {
+ return new RunLengthIntegerWriter(output, signed);
+ }
+ }
+
+ boolean isNewWriteFormat(StreamFactory writer) {
+ return writer.getVersion() != OrcFile.Version.V_0_11;
+ }
+
+ /**
+ * Add a new value to the column.
+ *
+ * @param obj The value to write to the column
+ * @throws IOException if an error occurs during add
+ */
+ void write(Object obj) throws IOException {
+ if (obj != null) {
+ indexStatistics.increment();
+ } else {
+ indexStatistics.setNull();
+ }
+ if (isPresent != null) {
+ isPresent.write(obj == null ? 0 : 1);
+ if (obj == null) {
+ foundNulls = true;
+ }
+ }
+ }
+
+ private void removeIsPresentPositions() {
+ for (int i = 0; i < rowIndex.getEntryCount(); ++i) {
+ RowIndexEntry.Builder entry = rowIndex.getEntryBuilder(i);
+ List<Long> positions = entry.getPositionsList();
+ // bit streams use 3 positions if uncompressed, 4 if compressed
+ positions = positions.subList(isCompressed ? 4 : 3, positions.size());
+ entry.clearPositions();
+ entry.addAllPositions(positions);
+ }
+ }
+
+ /**
+ * Write the stripe out to the file.
+ *
+ * @param builder the stripe footer that contains the information about the
+ * layout of the stripe. The TreeWriter is required to update
+ * the footer with its information.
+ * @param requiredIndexEntries the number of index entries that are
+ * required. this is to check to make sure the
+ * row index is well formed.
+ * @throws IOException if an error occurs during write
+ */
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ if (isPresent != null) {
+ isPresent.flush();
+
+ // if no nulls are found in a stream, then suppress the stream
+ if (!foundNulls) {
+ isPresentOutStream.suppress();
+ // since isPresent bitstream is suppressed, update the index to
+ // remove the positions of the isPresent stream
+ if (rowIndexStream != null) {
+ removeIsPresentPositions();
+ }
+ }
+ }
+
+ // merge stripe-level column statistics to file statistics and write it to
+ // stripe statistics
+ OrcProto.StripeStatistics.Builder stripeStatsBuilder = OrcProto.StripeStatistics.newBuilder();
+ writeStripeStatistics(stripeStatsBuilder, this);
+ stripeStatsBuilders.add(stripeStatsBuilder);
+
+ // reset the flag for next stripe
+ foundNulls = false;
+
+ builder.addColumns(getEncoding());
+ builder.setWriterTimezone(TimeZone.getDefault().getID());
+ if (rowIndexStream != null) {
+ if (rowIndex.getEntryCount() != requiredIndexEntries) {
+ throw new IllegalArgumentException("Column has wrong number of " +
+ "index entries found: " + rowIndex.getEntryCount() + " expected: " +
+ requiredIndexEntries);
+ }
+ rowIndex.build().writeTo(rowIndexStream);
+ rowIndexStream.flush();
+ }
+ rowIndex.clear();
+ rowIndexEntry.clear();
+
+ // write the bloom filter to out stream
+ if (bloomFilterStream != null) {
+ bloomFilterIndex.build().writeTo(bloomFilterStream);
+ bloomFilterStream.flush();
+ bloomFilterIndex.clear();
+ bloomFilterEntry.clear();
+ }
+ }
+
+ private void writeStripeStatistics(OrcProto.StripeStatistics.Builder builder,
+ TreeWriter treeWriter) {
+ treeWriter.fileStatistics.merge(treeWriter.stripeColStatistics);
+ builder.addColStats(treeWriter.stripeColStatistics.serialize().build());
+ treeWriter.stripeColStatistics.reset();
+ for (TreeWriter child : treeWriter.getChildrenWriters()) {
+ writeStripeStatistics(builder, child);
+ }
+ }
+
+ TreeWriter[] getChildrenWriters() {
+ return childrenWriters;
+ }
+
+ /**
+ * Get the encoding for this column.
+ *
+ * @return the information about the encoding of this column
+ */
+ OrcProto.ColumnEncoding getEncoding() {
+ return OrcProto.ColumnEncoding.newBuilder().setKind(
+ OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ /**
+ * Create a row index entry with the previous location and the current
+ * index statistics. Also merges the index statistics into the file
+ * statistics before they are cleared. Finally, it records the start of the
+ * next index and ensures all of the children columns also create an entry.
+ *
+ * @throws IOException if an error occurs during create
+ */
+ void createRowIndexEntry() throws IOException {
+ stripeColStatistics.merge(indexStatistics);
+ rowIndexEntry.setStatistics(indexStatistics.serialize());
+ indexStatistics.reset();
+ rowIndex.addEntry(rowIndexEntry);
+ rowIndexEntry.clear();
+ addBloomFilterEntry();
+ recordPosition(rowIndexPosition);
+ for (TreeWriter child : childrenWriters) {
+ child.createRowIndexEntry();
+ }
+ }
+
+ void addBloomFilterEntry() {
+ if (createBloomFilter) {
+ bloomFilterEntry.setNumHashFunctions(bloomFilter.getNumHashFunctions());
+ bloomFilterEntry.addAllBitset(Longs.asList(bloomFilter.getBitSet()));
+ bloomFilterIndex.addBloomFilter(bloomFilterEntry.build());
+ bloomFilter.reset();
+ bloomFilterEntry.clear();
+ }
+ }
+
+ /**
+ * Record the current position in each of this column's streams.
+ *
+ * @param recorder where should the locations be recorded
+ * @throws IOException if an error occurs during position recording
+ */
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ if (isPresent != null) {
+ isPresent.getPosition(recorder);
+ }
+ }
+
+ /**
+ * Estimate how much memory the writer is consuming excluding the streams.
+ *
+ * @return the number of bytes.
+ */
+ long estimateMemory() {
+ long result = 0;
+ for (TreeWriter child : childrenWriters) {
+ result += child.estimateMemory();
+ }
+ return result;
+ }
+
+ /**
+ * Handle the top level object write.
+ *
+ * This default method is used for all types except structs, which are the
+ * typical case. VectorizedRowBatch assumes the top level object is a
+ * struct, so we use the first column for all other types.
+ * @param batch the batch to write from
+ * @param offset the row to start on
+ * @param length the number of rows to write
+ * @throws IOException if an error occurs during write
+ */
+ void writeRootBatch(VectorizedRowBatch batch, int offset,
+ int length) throws IOException {
+ writeBatch(batch.cols[0], offset, length);
+ }
+
+ /**
+ * Write the values from the given vector from offset for length elements.
+ * @param vector the vector to write from
+ * @param offset the first value from the vector to write
+ * @param length the number of values from the vector to write
+ * @throws IOException if an error occurs during write
+ */
+ void writeBatch(ColumnVector vector, int offset,
+ int length) throws IOException {
+ if (vector.noNulls) {
+ if (isPresent != null) {
+ for (int i = 0; i < length; ++i) {
+ isPresent.write(1);
+ indexStatistics.increment();
+ }
+ }
+ } else {
+ if (vector.isRepeating) {
+ boolean isNull = vector.isNull[0];
+ if (isPresent != null) {
+ for (int i = 0; i < length; ++i) {
+ isPresent.write(isNull ? 0 : 1);
+ }
+ }
+ if (isNull) {
+ foundNulls = true;
+ indexStatistics.setNull();
+ } else {
+ indexStatistics.increment();
+ }
+ } else {
+ // count the number of non-null values
+ int nonNullCount = 0;
+ for(int i = 0; i < length; ++i) {
+ boolean isNull = vector.isNull[i + offset];
+ if (!isNull) {
+ nonNullCount += 1;
+ }
+ if (isPresent != null) {
+ isPresent.write(isNull ? 0 : 1);
+ }
+ }
+ indexStatistics.increment();
+ if (nonNullCount != length) {
+ foundNulls = true;
+ indexStatistics.setNull();
+ }
+ }
+ }
+ }
+ }
+
+ private static class BooleanTreeWriter extends TreeWriter {
+ private final BitFieldWriter writer;
+
+ BooleanTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ PositionedOutputStream out = writer.createStream(id,
+ OrcProto.Stream.Kind.DATA);
+ this.writer = new BitFieldWriter(out, 1);
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ boolean val = ((BooleanObjectInspector) inspector).get(obj);
+ indexStatistics.updateBoolean(val);
+ writer.write(val ? 1 : 0);
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ writer.flush();
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ writer.getPosition(recorder);
+ }
+ }
+
+ private static class ByteTreeWriter extends TreeWriter {
+ private final RunLengthByteWriter writer;
+
+ ByteTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ this.writer = new RunLengthByteWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.DATA));
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ byte val = ((ByteObjectInspector) inspector).get(obj);
+ indexStatistics.updateInteger(val);
+ if (createBloomFilter) {
+ bloomFilter.addLong(val);
+ }
+ writer.write(val);
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ writer.flush();
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ writer.getPosition(recorder);
+ }
+ }
+
+ private static class IntegerTreeWriter extends TreeWriter {
+ private final IntegerWriter writer;
+ private final ShortObjectInspector shortInspector;
+ private final IntObjectInspector intInspector;
+ private final LongObjectInspector longInspector;
+ private boolean isDirectV2 = true;
+
+ IntegerTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ OutStream out = writer.createStream(id,
+ OrcProto.Stream.Kind.DATA);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ this.writer = createIntegerWriter(out, true, isDirectV2, writer);
+ if (inspector instanceof IntObjectInspector) {
+ intInspector = (IntObjectInspector) inspector;
+ shortInspector = null;
+ longInspector = null;
+ } else {
+ intInspector = null;
+ if (inspector instanceof LongObjectInspector) {
+ longInspector = (LongObjectInspector) inspector;
+ shortInspector = null;
+ } else {
+ shortInspector = (ShortObjectInspector) inspector;
+ longInspector = null;
+ }
+ }
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ long val;
+ if (intInspector != null) {
+ val = intInspector.get(obj);
+ } else if (longInspector != null) {
+ val = longInspector.get(obj);
+ } else {
+ val = shortInspector.get(obj);
+ }
+ indexStatistics.updateInteger(val);
+ if (createBloomFilter) {
+ // integers are converted to longs in column statistics and during SARG evaluation
+ bloomFilter.addLong(val);
+ }
+ writer.write(val);
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ writer.flush();
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ writer.getPosition(recorder);
+ }
+ }
+
+ private static class FloatTreeWriter extends TreeWriter {
+ private final PositionedOutputStream stream;
+ private final SerializationUtils utils;
+
+ FloatTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ this.stream = writer.createStream(id,
+ OrcProto.Stream.Kind.DATA);
+ this.utils = new SerializationUtils();
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ float val = ((FloatObjectInspector) inspector).get(obj);
+ indexStatistics.updateDouble(val);
+ if (createBloomFilter) {
+ // floats are converted to doubles in column statistics and during SARG evaluation
+ bloomFilter.addDouble(val);
+ }
+ utils.writeFloat(stream, val);
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ stream.flush();
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ stream.getPosition(recorder);
+ }
+ }
+
+ private static class DoubleTreeWriter extends TreeWriter {
+ private final PositionedOutputStream stream;
+ private final SerializationUtils utils;
+
+ DoubleTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ this.stream = writer.createStream(id,
+ OrcProto.Stream.Kind.DATA);
+ this.utils = new SerializationUtils();
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ double val = ((DoubleObjectInspector) inspector).get(obj);
+ indexStatistics.updateDouble(val);
+ if (createBloomFilter) {
+ bloomFilter.addDouble(val);
+ }
+ utils.writeDouble(stream, val);
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ stream.flush();
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ stream.getPosition(recorder);
+ }
+ }
+
+ private static class StringTreeWriter extends TreeWriter {
+ private static final int INITIAL_DICTIONARY_SIZE = 4096;
+ private final OutStream stringOutput;
+ private final IntegerWriter lengthOutput;
+ private final IntegerWriter rowOutput;
+ private final StringRedBlackTree dictionary =
+ new StringRedBlackTree(INITIAL_DICTIONARY_SIZE);
+ private final DynamicIntArray rows = new DynamicIntArray();
+ private final PositionedOutputStream directStreamOutput;
+ private final IntegerWriter directLengthOutput;
+ private final List<OrcProto.RowIndexEntry> savedRowIndex =
+ new ArrayList<OrcProto.RowIndexEntry>();
+ private final boolean buildIndex;
+ private final List<Long> rowIndexValueCount = new ArrayList<Long>();
+ // If the number of keys in a dictionary is greater than this fraction of
+ //the total number of non-null rows, turn off dictionary encoding
+ private final float dictionaryKeySizeThreshold;
+ private boolean useDictionaryEncoding = true;
+ private boolean isDirectV2 = true;
+ private boolean doneDictionaryCheck;
+ private final boolean strideDictionaryCheck;
+
+ StringTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ stringOutput = writer.createStream(id,
+ OrcProto.Stream.Kind.DICTIONARY_DATA);
+ lengthOutput = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
+ rowOutput = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.DATA), false, isDirectV2, writer);
+ recordPosition(rowIndexPosition);
+ rowIndexValueCount.add(0L);
+ buildIndex = writer.buildIndex();
+ directStreamOutput = writer.createStream(id, OrcProto.Stream.Kind.DATA);
+ directLengthOutput = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
+ dictionaryKeySizeThreshold = writer.getConfiguration().getFloat(
+ HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.varname,
+ HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.defaultFloatVal);
+ strideDictionaryCheck = writer.getConfiguration().getBoolean(
+ HiveConf.ConfVars.HIVE_ORC_ROW_INDEX_STRIDE_DICTIONARY_CHECK.varname,
+ HiveConf.ConfVars.HIVE_ORC_ROW_INDEX_STRIDE_DICTIONARY_CHECK.defaultBoolVal);
+ doneDictionaryCheck = false;
+ }
+
+ /**
+ * Method to retrieve text values from the value object, which can be overridden
+ * by subclasses.
+ *
+ * @param obj value
+ * @return Text text value from obj
+ */
+ Text getTextValue(Object obj) {
+ return ((StringObjectInspector) inspector).getPrimitiveWritableObject(obj);
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ Text val = getTextValue(obj);
+ if (useDictionaryEncoding || !strideDictionaryCheck) {
+ rows.add(dictionary.add(val));
+ } else {
+ // write data and length
+ directStreamOutput.write(val.getBytes(), 0, val.getLength());
+ directLengthOutput.write(val.getLength());
+ }
+ indexStatistics.updateString(val);
+ if (createBloomFilter) {
+ bloomFilter.addBytes(val.getBytes(), val.getLength());
+ }
+ }
+ }
+
+ private boolean checkDictionaryEncoding() {
+ if (!doneDictionaryCheck) {
+ // Set the flag indicating whether or not to use dictionary encoding
+ // based on whether or not the fraction of distinct keys over number of
+ // non-null rows is less than the configured threshold
+ float ratio = rows.size() > 0 ? (float) (dictionary.size()) / rows.size() : 0.0f;
+ useDictionaryEncoding = !isDirectV2 || ratio <= dictionaryKeySizeThreshold;
+ doneDictionaryCheck = true;
+ }
+ return useDictionaryEncoding;
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ // if rows in stripe is less than dictionaryCheckAfterRows, dictionary
+ // checking would not have happened. So do it again here.
+ checkDictionaryEncoding();
+
+ if (useDictionaryEncoding) {
+ flushDictionary();
+ } else {
+ // flushout any left over entries from dictionary
+ if (rows.size() > 0) {
+ flushDictionary();
+ }
+
+ // suppress the stream for every stripe if dictionary is disabled
+ stringOutput.suppress();
+ }
+
+ // we need to build the rowindex before calling super, since it
+ // writes it out.
+ super.writeStripe(builder, requiredIndexEntries);
+ stringOutput.flush();
+ lengthOutput.flush();
+ rowOutput.flush();
+ directStreamOutput.flush();
+ directLengthOutput.flush();
+ // reset all of the fields to be ready for the next stripe.
+ dictionary.clear();
+ savedRowIndex.clear();
+ rowIndexValueCount.clear();
+ recordPosition(rowIndexPosition);
+ rowIndexValueCount.add(0L);
+
+ if (!useDictionaryEncoding) {
+ // record the start positions of first index stride of next stripe i.e
+ // beginning of the direct streams when dictionary is disabled
+ recordDirectStreamPosition();
+ }
+ }
+
+ private void flushDictionary() throws IOException {
+ final int[] dumpOrder = new int[dictionary.size()];
+
+ if (useDictionaryEncoding) {
+ // Write the dictionary by traversing the red-black tree writing out
+ // the bytes and lengths; and creating the map from the original order
+ // to the final sorted order.
+
+ dictionary.visit(new StringRedBlackTree.Visitor() {
+ private int currentId = 0;
+
+ @Override
+ public void visit(StringRedBlackTree.VisitorContext context
+ ) throws IOException {
+ context.writeBytes(stringOutput);
+ lengthOutput.write(context.getLength());
+ dumpOrder[context.getOriginalPosition()] = currentId++;
+ }
+ });
+ } else {
+ // for direct encoding, we don't want the dictionary data stream
+ stringOutput.suppress();
+ }
+ int length = rows.size();
+ int rowIndexEntry = 0;
+ OrcProto.RowIndex.Builder rowIndex = getRowIndex();
+ Text text = new Text();
+ // write the values translated into the dump order.
+ for (int i = 0; i <= length; ++i) {
+ // now that we are writing out the row values, we can finalize the
+ // row index
+ if (buildIndex) {
+ while (i == rowIndexValueCount.get(rowIndexEntry) && rowIndexEntry < savedRowIndex.size()) {
+ OrcProto.RowIndexEntry.Builder base =
+ savedRowIndex.get(rowIndexEntry++).toBuilder();
+ if (useDictionaryEncoding) {
+ rowOutput.getPosition(new RowIndexPositionRecorder(base));
+ } else {
+ PositionRecorder posn = new RowIndexPositionRecorder(base);
+ directStreamOutput.getPosition(posn);
+ directLengthOutput.getPosition(posn);
+ }
+ rowIndex.addEntry(base.build());
+ }
+ }
+ if (i != length) {
+ if (useDictionaryEncoding) {
+ rowOutput.write(dumpOrder[rows.get(i)]);
+ } else {
+ dictionary.getText(text, rows.get(i));
+ directStreamOutput.write(text.getBytes(), 0, text.getLength());
+ directLengthOutput.write(text.getLength());
+ }
+ }
+ }
+ rows.clear();
+ }
+
+ @Override
+ OrcProto.ColumnEncoding getEncoding() {
+ // Returns the encoding used for the last call to writeStripe
+ if (useDictionaryEncoding) {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder().setKind(
+ OrcProto.ColumnEncoding.Kind.DICTIONARY_V2).setDictionarySize(dictionary.size()).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder().setKind(
+ OrcProto.ColumnEncoding.Kind.DICTIONARY).setDictionarySize(dictionary.size()).build();
+ } else {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder().setKind(
+ OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder().setKind(
+ OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+ }
+
+ /**
+ * This method doesn't call the super method, because unlike most of the
+ * other TreeWriters, this one can't record the position in the streams
+ * until the stripe is being flushed. Therefore it saves all of the entries
+ * and augments them with the final information as the stripe is written.
+ *
+ * @throws IOException if an error occurs on create
+ */
+ @Override
+ void createRowIndexEntry() throws IOException {
+ getStripeStatistics().merge(indexStatistics);
+ OrcProto.RowIndexEntry.Builder rowIndexEntry = getRowIndexEntry();
+ rowIndexEntry.setStatistics(indexStatistics.serialize());
+ indexStatistics.reset();
+ OrcProto.RowIndexEntry base = rowIndexEntry.build();
+ savedRowIndex.add(base);
+ rowIndexEntry.clear();
+ addBloomFilterEntry();
+ recordPosition(rowIndexPosition);
+ rowIndexValueCount.add(Long.valueOf(rows.size()));
+ if (strideDictionaryCheck) {
+ checkDictionaryEncoding();
+ }
+ if (!useDictionaryEncoding) {
+ if (rows.size() > 0) {
+ flushDictionary();
+ // just record the start positions of next index stride
+ recordDirectStreamPosition();
+ } else {
+ // record the start positions of next index stride
+ recordDirectStreamPosition();
+ getRowIndex().addEntry(base);
+ }
+ }
+ }
+
+ private void recordDirectStreamPosition() throws IOException {
+ directStreamOutput.getPosition(rowIndexPosition);
+ directLengthOutput.getPosition(rowIndexPosition);
+ }
+
+ @Override
+ long estimateMemory() {
+ return rows.getSizeInBytes() + dictionary.getSizeInBytes();
+ }
+ }
+
+ /**
+ * Under the covers, char is written to ORC the same way as string.
+ */
+ private static class CharTreeWriter extends StringTreeWriter {
+
+ CharTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ }
+
+ /**
+ * Override base class implementation to support char values.
+ */
+ @Override
+ Text getTextValue(Object obj) {
+ return (((HiveCharObjectInspector) inspector)
+ .getPrimitiveWritableObject(obj)).getTextValue();
+ }
+ }
+
+ /**
+ * Under the covers, varchar is written to ORC the same way as string.
+ */
+ private static class VarcharTreeWriter extends StringTreeWriter {
+
+ VarcharTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ }
+
+ /**
+ * Override base class implementation to support varchar values.
+ */
+ @Override
+ Text getTextValue(Object obj) {
+ return (((HiveVarcharObjectInspector) inspector)
+ .getPrimitiveWritableObject(obj)).getTextValue();
+ }
+ }
+
+ private static class BinaryTreeWriter extends TreeWriter {
+ private final PositionedOutputStream stream;
+ private final IntegerWriter length;
+ private boolean isDirectV2 = true;
+
+ BinaryTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ this.stream = writer.createStream(id,
+ OrcProto.Stream.Kind.DATA);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ this.length = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ BytesWritable val =
+ ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj);
+ stream.write(val.getBytes(), 0, val.getLength());
+ length.write(val.getLength());
+ indexStatistics.updateBinary(val);
+ if (createBloomFilter) {
+ bloomFilter.addBytes(val.getBytes(), val.getLength());
+ }
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ stream.flush();
+ length.flush();
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ stream.getPosition(recorder);
+ length.getPosition(recorder);
+ }
+ }
+
+ static final int MILLIS_PER_SECOND = 1000;
+ static final String BASE_TIMESTAMP_STRING = "2015-01-01 00:00:00";
+
+ private static class TimestampTreeWriter extends TreeWriter {
+ private final IntegerWriter seconds;
+ private final IntegerWriter nanos;
+ private final boolean isDirectV2;
+ private final long base_timestamp;
+
+ TimestampTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ this.seconds = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.DATA), true, isDirectV2, writer);
+ this.nanos = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.SECONDARY), false, isDirectV2, writer);
+ recordPosition(rowIndexPosition);
+ // for unit tests to set different time zones
+ this.base_timestamp = Timestamp.valueOf(BASE_TIMESTAMP_STRING).getTime() / MILLIS_PER_SECOND;
+ }
+
+ @Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ Timestamp val =
+ ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(obj);
+ indexStatistics.updateTimestamp(val);
+ seconds.write((val.getTime() / MILLIS_PER_SECOND) - base_timestamp);
+ nanos.write(formatNanos(val.getNanos()));
+ if (createBloomFilter) {
+ bloomFilter.addLong(val.getTime());
+ }
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ seconds.flush();
+ nanos.flush();
+ recordPosition(rowIndexPosition);
+ }
+
+ private static long formatNanos(int nanos) {
+ if (nanos == 0) {
+ return 0;
+ } else if (nanos % 100 != 0) {
+ return ((long) nanos) << 3;
+ } else {
+ nanos /= 100;
+ int trailingZeros = 1;
+ while (nanos % 10 == 0 && trailingZeros < 7) {
+ nanos /= 10;
+ trailingZeros += 1;
+ }
+ return ((long) nanos) << 3 | trailingZeros;
+ }
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ seconds.getPosition(recorder);
+ nanos.getPosition(recorder);
+ }
+ }
+
+ private static class DateTreeWriter extends TreeWriter {
+ private final IntegerWriter writer;
+ private final boolean isDirectV2;
+
+ DateTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ OutStream out = writer.createStream(id,
+ OrcProto.Stream.Kind.DATA);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ this.writer = createIntegerWriter(out, true, isDirectV2, writer);
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ // Using the Writable here as it's used directly for writing as well as for stats.
+ DateWritable val = ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj);
+ indexStatistics.updateDate(val);
+ writer.write(val.getDays());
+ if (createBloomFilter) {
+ bloomFilter.addLong(val.getDays());
+ }
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ writer.flush();
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ writer.getPosition(recorder);
+ }
+
+ @Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+ }
+
+ private static class DecimalTreeWriter extends TreeWriter {
+ private final PositionedOutputStream valueStream;
+ private final IntegerWriter scaleStream;
+ private final boolean isDirectV2;
+
+ DecimalTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ valueStream = writer.createStream(id, OrcProto.Stream.Kind.DATA);
+ this.scaleStream = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.SECONDARY), true, isDirectV2, writer);
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ HiveDecimal decimal = ((HiveDecimalObjectInspector) inspector).getPrimitiveJavaObject(obj);
+ if (decimal == null) {
+ return;
+ }
+ SerializationUtils.writeBigInteger(valueStream,
+ decimal.unscaledValue());
+ scaleStream.write(decimal.scale());
+ indexStatistics.updateDecimal(decimal);
+ if (createBloomFilter) {
+ bloomFilter.addString(decimal.toString());
+ }
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ valueStream.flush();
+ scaleStream.flush();
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ valueStream.getPosition(recorder);
+ scaleStream.getPosition(recorder);
+ }
+ }
+
+ private static class StructTreeWriter extends TreeWriter {
+ private final List<? extends StructField> fields;
+
+ StructTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ StructObjectInspector structObjectInspector =
+ (StructObjectInspector) inspector;
+ fields = structObjectInspector.getAllStructFieldRefs();
+ childrenWriters = new TreeWriter[fields.size()];
+ for (int i = 0; i < childrenWriters.length; ++i) {
+ childrenWriters[i] = createTreeWriter(
+ fields.get(i).getFieldObjectInspector(), writer, true);
+ }
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ StructObjectInspector insp = (StructObjectInspector) inspector;
+ for (int i = 0; i < fields.size(); ++i) {
+ StructField field = fields.get(i);
+ TreeWriter writer = childrenWriters[i];
+ writer.write(insp.getStructFieldData(obj, field));
+ }
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ for (TreeWriter child : childrenWriters) {
+ child.writeStripe(builder, requiredIndexEntries);
+ }
+ recordPosition(rowIndexPosition);
+ }
+ }
+
+ private static class ListTreeWriter extends TreeWriter {
+ private final IntegerWriter lengths;
+ private final boolean isDirectV2;
+
+ ListTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ ListObjectInspector listObjectInspector = (ListObjectInspector) inspector;
+ childrenWriters = new TreeWriter[1];
+ childrenWriters[0] =
+ createTreeWriter(listObjectInspector.getListElementObjectInspector(),
+ writer, true);
+ lengths = createIntegerWriter(writer.createStream(columnId,
+ OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ ListObjectInspector insp = (ListObjectInspector) inspector;
+ int len = insp.getListLength(obj);
+ lengths.write(len);
+ if (createBloomFilter) {
+ bloomFilter.addLong(len);
+ }
+ for (int i = 0; i < len; ++i) {
+ childrenWriters[0].write(insp.getListElement(obj, i));
+ }
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ lengths.flush();
+ for (TreeWriter child : childrenWriters) {
+ child.writeStripe(builder, requiredIndexEntries);
+ }
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ lengths.getPosition(recorder);
+ }
+ }
+
+ private static class MapTreeWriter extends TreeWriter {
+ private final IntegerWriter lengths;
+ private final boolean isDirectV2;
+
+ MapTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ MapObjectInspector insp = (MapObjectInspector) inspector;
+ childrenWriters = new TreeWriter[2];
+ childrenWriters[0] =
+ createTreeWriter(insp.getMapKeyObjectInspector(), writer, true);
+ childrenWriters[1] =
+ createTreeWriter(insp.getMapValueObjectInspector(), writer, true);
+ lengths = createIntegerWriter(writer.createStream(columnId,
+ OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ MapObjectInspector insp = (MapObjectInspector) inspector;
+ // this sucks, but it will have to do until we can get a better
+ // accessor in the MapObjectInspector.
+ Map<?, ?> valueMap = insp.getMap(obj);
+ lengths.write(valueMap.size());
+ if (createBloomFilter) {
+ bloomFilter.addLong(valueMap.size());
+ }
+ for (Map.Entry<?, ?> entry : valueMap.entrySet()) {
+ childrenWriters[0].write(entry.getKey());
+ childrenWriters[1].write(entry.getValue());
+ }
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ lengths.flush();
+ for (TreeWriter child : childrenWriters) {
+ child.writeStripe(builder, requiredIndexEntries);
+ }
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ lengths.getPosition(recorder);
+ }
+ }
+
+ private static class UnionTreeWriter extends TreeWriter {
+ private final RunLengthByteWriter tags;
+
+ UnionTreeWriter(int columnId,
+ ObjectInspector inspector,
+ StreamFactory writer,
+ boolean nullable) throws IOException {
+ super(columnId, inspector, writer, nullable);
+ UnionObjectInspector insp = (UnionObjectInspector) inspector;
+ List<ObjectInspector> choices = insp.getObjectInspectors();
+ childrenWriters = new TreeWriter[choices.size()];
+ for (int i = 0; i < childrenWriters.length; ++i) {
+ childrenWriters[i] = createTreeWriter(choices.get(i), writer, true);
+ }
+ tags =
+ new RunLengthByteWriter(writer.createStream(columnId,
+ OrcProto.Stream.Kind.DATA));
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void write(Object obj) throws IOException {
+ super.write(obj);
+ if (obj != null) {
+ UnionObjectInspector insp = (UnionObjectInspector) inspector;
+ byte tag = insp.getTag(obj);
+ tags.write(tag);
+ if (createBloomFilter) {
+ bloomFilter.addLong(tag);
+ }
+ childrenWriters[tag].write(insp.getField(obj));
+ }
+ }
+
+ @Override
+ void writeStripe(OrcProto.StripeFooter.Builder builder,
+ int requiredIndexEntries) throws IOException {
+ super.writeStripe(builder, requiredIndexEntries);
+ tags.flush();
+ for (TreeWriter child : childrenWriters) {
+ child.writeStripe(builder, requiredIndexEntries);
+ }
+ recordPosition(rowIndexPosition);
+ }
+
+ @Override
+ void recordPosition(PositionRecorder recorder) throws IOException {
+ super.recordPosition(recorder);
+ tags.getPosition(recorder);
+ }
+ }
+
+ private static TreeWriter createTreeWriter(ObjectInspector inspector,
+ StreamFactory streamFactory,
+ boolean nullable) throws IOException {
+ switch (inspector.getCategory()) {
+ case PRIMITIVE:
+ switch (((PrimitiveObjectInspector) inspector).getPrimitiveCategory()) {
+ case BOOLEAN:
+ return new BooleanTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case BYTE:
+ return new ByteTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case SHORT:
+ case INT:
+ case LONG:
+ return new IntegerTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case FLOAT:
+ return new FloatTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case DOUBLE:
+ return new DoubleTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case STRING:
+ return new StringTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case CHAR:
+ return new CharTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case VARCHAR:
+ return new VarcharTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case BINARY:
+ return new BinaryTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case TIMESTAMP:
+ return new TimestampTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case DATE:
+ return new DateTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ case DECIMAL:
+ return new DecimalTreeWriter(streamFactory.getNextColumnId(),
+ inspector, streamFactory, nullable);
+ default:
+ throw new IllegalArgumentException("Bad primitive category " +
+ ((PrimitiveObjectInspector) inspector).getPrimitiveCategory());
+ }
+ case STRUCT:
+ return new StructTreeWriter(streamFactory.getNextColumnId(), inspector,
+ streamFactory, nullable);
+ case MAP:
+ return new MapTreeWriter(streamFactory.getNextColumnId(), inspector,
+ streamFactory, nullable);
+ case LIST:
+ return new ListTreeWriter(streamFactory.getNextColumnId(), inspector,
+ streamFactory, nullable);
+ case UNION:
+ return new UnionTreeWriter(streamFactory.getNextColumnId(), inspector,
+ streamFactory, nullable);
+ default:
+ throw new IllegalArgumentException("Bad category: " +
+ inspector.getCategory());
+ }
+ }
+
+ private static void writeTypes(OrcProto.Footer.Builder builder,
+ TreeWriter treeWriter) {
+ OrcProto.Type.Builder type = OrcProto.Type.newBuilder();
+ switch (treeWriter.inspector.getCategory()) {
+ case PRIMITIVE:
+ switch (((PrimitiveObjectInspector) treeWriter.inspector).getPrimitiveCategory()) {
+ case BOOLEAN:
+ type.setKind(OrcProto.Type.Kind.BOOLEAN);
+ break;
+ case BYTE:
+ type.setKind(OrcProto.Type.Kind.BYTE);
+ break;
+ case SHORT:
+ type.setKind(OrcProto.Type.Kind.SHORT);
+ break;
+ case INT:
+ type.setKind(OrcProto.Type.Kind.INT);
+ break;
+ case LONG:
+ type.setKind(OrcProto.Type.Kind.LONG);
+ break;
+ case FLOAT:
+ type.setKind(OrcProto.Type.Kind.FLOAT);
+ break;
+ case DOUBLE:
+ type.setKind(OrcProto.Type.Kind.DOUBLE);
+
<TRUNCATED>
[3/3] nifi git commit: NIFI-1663: Add ConvertAvroToORC processor
Posted by oz...@apache.org.
NIFI-1663: Add ConvertAvroToORC processor
This closes #727
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d9720239
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d9720239
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d9720239
Branch: refs/heads/master
Commit: d9720239f52261225191b28e7804d0ed2636ee0f
Parents: bb24312
Author: Matt Burgess <ma...@apache.org>
Authored: Tue Jul 26 23:25:11 2016 -0400
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Wed Aug 10 12:37:15 2016 -0400
----------------------------------------------------------------------
.../nifi-hive-processors/pom.xml | 9 -
.../hadoop/hive/ql/io/orc/NiFiOrcUtils.java | 466 +++
.../hive/ql/io/orc/OrcFlowFileWriter.java | 2650 ++++++++++++++++++
.../nifi/processors/hive/ConvertAvroToORC.java | 289 ++
.../org.apache.nifi.processor.Processor | 1 +
.../processors/hive/TestConvertAvroToORC.java | 214 ++
.../processors/hive/TestPutHiveStreaming.java | 10 +-
.../apache/nifi/util/orc/TestNiFiOrcUtils.java | 371 +++
8 files changed, 3996 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/d9720239/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/pom.xml
index 9ef6fc4..02face9 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/pom.xml
@@ -45,10 +45,6 @@
<version>${hive.version}</version>
<exclusions>
<exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-orc</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
@@ -192,11 +188,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
http://git-wip-us.apache.org/repos/asf/nifi/blob/d9720239/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
new file mode 100644
index 0000000..78430bf
--- /dev/null
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.util.Utf8;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.io.filters.BloomFilterIO;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_ORC_DEFAULT_BLOCK_PADDING;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_ORC_DEFAULT_BLOCK_SIZE;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_ORC_DEFAULT_ROW_INDEX_STRIDE;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_ORC_WRITE_FORMAT;
+
+/**
+ * Utility methods for ORC support (conversion from Avro, conversion to Hive types, e.g.
+ */
+public class NiFiOrcUtils {
+
+ public static Object convertToORCObject(TypeInfo typeInfo, Object o) {
+ if (o != null) {
+ if (typeInfo instanceof UnionTypeInfo) {
+ OrcUnion union = new OrcUnion();
+ // Need to find which of the union types correspond to the primitive object
+ TypeInfo objectTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(
+ ObjectInspectorFactory.getReflectionObjectInspector(o.getClass(), ObjectInspectorFactory.ObjectInspectorOptions.JAVA));
+ List<TypeInfo> unionTypeInfos = ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos();
+
+ int index = 0;
+ while (index < unionTypeInfos.size() && !unionTypeInfos.get(index).equals(objectTypeInfo)) {
+ index++;
+ }
+ if (index < unionTypeInfos.size()) {
+ union.set((byte) index, convertToORCObject(objectTypeInfo, o));
+ } else {
+ throw new IllegalArgumentException("Object Type for class " + o.getClass().getName() + " not in Union declaration");
+ }
+ return union;
+ }
+ if (o instanceof Integer) {
+ return new IntWritable((int) o);
+ }
+ if (o instanceof Boolean) {
+ return new BooleanWritable((boolean) o);
+ }
+ if (o instanceof Long) {
+ return new LongWritable((long) o);
+ }
+ if (o instanceof Float) {
+ return new FloatWritable((float) o);
+ }
+ if (o instanceof Double) {
+ return new DoubleWritable((double) o);
+ }
+ if (o instanceof String || o instanceof Utf8 || o instanceof GenericData.EnumSymbol) {
+ return new Text(o.toString());
+ }
+ if (o instanceof ByteBuffer) {
+ return new BytesWritable(((ByteBuffer) o).array());
+ }
+ if (o instanceof int[]) {
+ int[] intArray = (int[]) o;
+ return Arrays.stream(intArray)
+ .mapToObj((element) -> convertToORCObject(TypeInfoFactory.getPrimitiveTypeInfo("int"), element))
+ .collect(Collectors.toList());
+ }
+ if (o instanceof long[]) {
+ long[] longArray = (long[]) o;
+ return Arrays.stream(longArray)
+ .mapToObj((element) -> convertToORCObject(TypeInfoFactory.getPrimitiveTypeInfo("bigint"), element))
+ .collect(Collectors.toList());
+ }
+ if (o instanceof float[]) {
+ float[] floatArray = (float[]) o;
+ return IntStream.range(0, floatArray.length)
+ .mapToDouble(i -> floatArray[i])
+ .mapToObj((element) -> convertToORCObject(TypeInfoFactory.getPrimitiveTypeInfo("float"), (float) element))
+ .collect(Collectors.toList());
+ }
+ if (o instanceof double[]) {
+ double[] doubleArray = (double[]) o;
+ return Arrays.stream(doubleArray)
+ .mapToObj((element) -> convertToORCObject(TypeInfoFactory.getPrimitiveTypeInfo("double"), element))
+ .collect(Collectors.toList());
+ }
+ if (o instanceof boolean[]) {
+ boolean[] booleanArray = (boolean[]) o;
+ return IntStream.range(0, booleanArray.length)
+ .map(i -> booleanArray[i] ? 1 : 0)
+ .mapToObj((element) -> convertToORCObject(TypeInfoFactory.getPrimitiveTypeInfo("boolean"), element == 1))
+ .collect(Collectors.toList());
+ }
+ if (o instanceof GenericData.Array) {
+ GenericData.Array array = ((GenericData.Array) o);
+ // The type information in this case is interpreted as a List
+ TypeInfo listTypeInfo = ((ListTypeInfo) typeInfo).getListElementTypeInfo();
+ return array.stream().map((element) -> convertToORCObject(listTypeInfo, element)).collect(Collectors.toList());
+ }
+ if (o instanceof List) {
+ return o;
+ }
+ if (o instanceof Map) {
+ MapWritable mapWritable = new MapWritable();
+ TypeInfo keyInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo();
+ TypeInfo valueInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo();
+ // Unions are not allowed as key/value types, so if we convert the key and value objects,
+ // they should return Writable objects
+ ((Map) o).forEach((key, value) -> {
+ Object keyObject = convertToORCObject(keyInfo, key);
+ Object valueObject = convertToORCObject(valueInfo, value);
+ if (keyObject == null
+ || !(keyObject instanceof Writable)
+ || !(valueObject instanceof Writable)
+ ) {
+ throw new IllegalArgumentException("Maps may only contain Writable types, and the key cannot be null");
+ }
+ mapWritable.put((Writable) keyObject, (Writable) valueObject);
+ });
+ return mapWritable;
+ }
+
+ }
+ return null;
+ }
+
+
+ /**
+ * Create an object of OrcStruct given a TypeInfo and a list of objects
+ *
+ * @param typeInfo The TypeInfo object representing the ORC record schema
+ * @param objs ORC objects/Writables
+ * @return an OrcStruct containing the specified objects for the specified schema
+ */
+ public static OrcStruct createOrcStruct(TypeInfo typeInfo, Object... objs) {
+ SettableStructObjectInspector oi = (SettableStructObjectInspector) OrcStruct
+ .createObjectInspector(typeInfo);
+ List<StructField> fields = (List<StructField>) oi.getAllStructFieldRefs();
+ OrcStruct result = (OrcStruct) oi.create();
+ result.setNumFields(fields.size());
+ for (int i = 0; i < fields.size(); i++) {
+ oi.setStructFieldData(result, fields.get(i), objs[i]);
+ }
+ return result;
+ }
+
+ public static String normalizeHiveTableName(String name) {
+ return name.replaceAll("[\\. ]", "_");
+ }
+
+ public static String generateHiveDDL(Schema avroSchema, String tableName) {
+ Schema.Type schemaType = avroSchema.getType();
+ StringBuilder sb = new StringBuilder("CREATE EXTERNAL TABLE IF NOT EXISTS ");
+ sb.append(tableName);
+ sb.append(" (");
+ if (Schema.Type.RECORD.equals(schemaType)) {
+ List<String> hiveColumns = new ArrayList<>();
+ List<Schema.Field> fields = avroSchema.getFields();
+ if (fields != null) {
+ hiveColumns.addAll(
+ fields.stream().map(field -> field.name() + " " + getHiveTypeFromAvroType(field.schema())).collect(Collectors.toList()));
+ }
+ sb.append(StringUtils.join(hiveColumns, ", "));
+ sb.append(") STORED AS ORC");
+ return sb.toString();
+ } else {
+ throw new IllegalArgumentException("Avro schema is of type " + schemaType.getName() + ", not RECORD");
+ }
+ }
+
+
+ public static TypeInfo getOrcField(Schema fieldSchema) throws IllegalArgumentException {
+ Schema.Type fieldType = fieldSchema.getType();
+
+ switch (fieldType) {
+ case INT:
+ case LONG:
+ case BOOLEAN:
+ case BYTES:
+ case DOUBLE:
+ case FLOAT:
+ case STRING:
+ return getPrimitiveOrcTypeFromPrimitiveAvroType(fieldType);
+
+ case UNION:
+ List<Schema> unionFieldSchemas = fieldSchema.getTypes();
+
+ if (unionFieldSchemas != null) {
+ // Ignore null types in union
+ List<TypeInfo> orcFields = unionFieldSchemas.stream().filter(
+ unionFieldSchema -> !Schema.Type.NULL.equals(unionFieldSchema.getType()))
+ .map(NiFiOrcUtils::getOrcField)
+ .collect(Collectors.toList());
+
+ // Flatten the field if the union only has one non-null element
+ if (orcFields.size() == 1) {
+ return orcFields.get(0);
+ } else {
+ return TypeInfoFactory.getUnionTypeInfo(orcFields);
+ }
+ }
+ return null;
+
+ case ARRAY:
+ return TypeInfoFactory.getListTypeInfo(getOrcField(fieldSchema.getElementType()));
+
+ case MAP:
+ return TypeInfoFactory.getMapTypeInfo(
+ getPrimitiveOrcTypeFromPrimitiveAvroType(Schema.Type.STRING),
+ getOrcField(fieldSchema.getValueType()));
+
+ case RECORD:
+ List<Schema.Field> avroFields = fieldSchema.getFields();
+ if (avroFields != null) {
+ List<String> orcFieldNames = new ArrayList<>(avroFields.size());
+ List<TypeInfo> orcFields = new ArrayList<>(avroFields.size());
+ avroFields.forEach(avroField -> {
+ String fieldName = avroField.name();
+ orcFieldNames.add(fieldName);
+ orcFields.add(getOrcField(avroField.schema()));
+ });
+ return TypeInfoFactory.getStructTypeInfo(orcFieldNames, orcFields);
+ }
+ return null;
+
+ case ENUM:
+ // An enum value is just a String for ORC/Hive
+ return getPrimitiveOrcTypeFromPrimitiveAvroType(Schema.Type.STRING);
+
+ default:
+ throw new IllegalArgumentException("Did not recognize Avro type " + fieldType.getName());
+ }
+
+ }
+
+ public static Schema.Type getAvroSchemaTypeOfObject(Object o) {
+ if (o == null) {
+ return Schema.Type.NULL;
+ } else if (o instanceof Integer) {
+ return Schema.Type.INT;
+ } else if (o instanceof Long) {
+ return Schema.Type.LONG;
+ } else if (o instanceof Boolean) {
+ return Schema.Type.BOOLEAN;
+ } else if (o instanceof byte[]) {
+ return Schema.Type.BYTES;
+ } else if (o instanceof Float) {
+ return Schema.Type.FLOAT;
+ } else if (o instanceof Double) {
+ return Schema.Type.DOUBLE;
+ } else if (o instanceof Enum) {
+ return Schema.Type.ENUM;
+ } else if (o instanceof Object[]) {
+ return Schema.Type.ARRAY;
+ } else if (o instanceof List) {
+ return Schema.Type.ARRAY;
+ } else if (o instanceof Map) {
+ return Schema.Type.MAP;
+ } else {
+ throw new IllegalArgumentException("Object of class " + o.getClass() + " is not a supported Avro Type");
+ }
+ }
+
+ public static TypeInfo getPrimitiveOrcTypeFromPrimitiveAvroType(Schema.Type avroType) throws IllegalArgumentException {
+ if (avroType == null) {
+ throw new IllegalArgumentException("Avro type is null");
+ }
+ switch (avroType) {
+ case INT:
+ return TypeInfoFactory.getPrimitiveTypeInfo("int");
+ case LONG:
+ return TypeInfoFactory.getPrimitiveTypeInfo("bigint");
+ case BOOLEAN:
+ return TypeInfoFactory.getPrimitiveTypeInfo("boolean");
+ case BYTES:
+ return TypeInfoFactory.getPrimitiveTypeInfo("binary");
+ case DOUBLE:
+ return TypeInfoFactory.getPrimitiveTypeInfo("double");
+ case FLOAT:
+ return TypeInfoFactory.getPrimitiveTypeInfo("float");
+ case STRING:
+ return TypeInfoFactory.getPrimitiveTypeInfo("string");
+ default:
+ throw new IllegalArgumentException("Avro type " + avroType.getName() + " is not a primitive type");
+ }
+ }
+
+ public static String getHiveTypeFromAvroType(Schema avroSchema) {
+ if (avroSchema == null) {
+ throw new IllegalArgumentException("Avro schema is null");
+ }
+
+ Schema.Type avroType = avroSchema.getType();
+
+ switch (avroType) {
+ case INT:
+ return "INT";
+ case LONG:
+ return "BIGINT";
+ case BOOLEAN:
+ return "BOOLEAN";
+ case BYTES:
+ return "BINARY";
+ case DOUBLE:
+ return "DOUBLE";
+ case FLOAT:
+ return "FLOAT";
+ case STRING:
+ case ENUM:
+ return "STRING";
+ case UNION:
+ List<Schema> unionFieldSchemas = avroSchema.getTypes();
+ if (unionFieldSchemas != null) {
+ List<String> hiveFields = new ArrayList<>();
+ for (Schema unionFieldSchema : unionFieldSchemas) {
+ Schema.Type unionFieldSchemaType = unionFieldSchema.getType();
+ // Ignore null types in union
+ if (!Schema.Type.NULL.equals(unionFieldSchemaType)) {
+ hiveFields.add(getHiveTypeFromAvroType(unionFieldSchema));
+ }
+ }
+ // Flatten the field if the union only has one non-null element
+ return (hiveFields.size() == 1)
+ ? hiveFields.get(0)
+ : "UNIONTYPE<" + StringUtils.join(hiveFields, ", ") + ">";
+
+ }
+ break;
+ case MAP:
+ return "MAP<STRING, " + getHiveTypeFromAvroType(avroSchema.getValueType()) + ">";
+ case ARRAY:
+ return "ARRAY<" + getHiveTypeFromAvroType(avroSchema.getElementType()) + ">";
+ case RECORD:
+ List<Schema.Field> recordFields = avroSchema.getFields();
+ if (recordFields != null) {
+ List<String> hiveFields = recordFields.stream().map(
+ recordField -> recordField.name() + ":" + getHiveTypeFromAvroType(recordField.schema())).collect(Collectors.toList());
+ return "STRUCT<" + StringUtils.join(hiveFields, ", ") + ">";
+ }
+ break;
+ default:
+ break;
+ }
+
+ throw new IllegalArgumentException("Error converting Avro type " + avroType.getName() + " to Hive type");
+ }
+
+
+ public static OrcFlowFileWriter createWriter(OutputStream flowFileOutputStream,
+ Path path,
+ Configuration conf,
+ TypeInfo orcSchema,
+ long stripeSize,
+ CompressionKind compress,
+ int bufferSize) throws IOException {
+
+ int rowIndexStride = HiveConf.getIntVar(conf, HIVE_ORC_DEFAULT_ROW_INDEX_STRIDE);
+
+ boolean addBlockPadding = HiveConf.getBoolVar(conf, HIVE_ORC_DEFAULT_BLOCK_PADDING);
+
+ String versionName = HiveConf.getVar(conf, HIVE_ORC_WRITE_FORMAT);
+ OrcFile.Version versionValue = (versionName == null)
+ ? OrcFile.Version.CURRENT
+ : OrcFile.Version.byName(versionName);
+
+ OrcFile.EncodingStrategy encodingStrategy;
+ String enString = conf.get(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname);
+ if (enString == null) {
+ encodingStrategy = OrcFile.EncodingStrategy.SPEED;
+ } else {
+ encodingStrategy = OrcFile.EncodingStrategy.valueOf(enString);
+ }
+
+ OrcFile.CompressionStrategy compressionStrategy;
+ String compString = conf.get(HiveConf.ConfVars.HIVE_ORC_COMPRESSION_STRATEGY.varname);
+ if (compString == null) {
+ compressionStrategy = OrcFile.CompressionStrategy.SPEED;
+ } else {
+ compressionStrategy = OrcFile.CompressionStrategy.valueOf(compString);
+ }
+
+ float paddingTolerance;
+ paddingTolerance = conf.getFloat(HiveConf.ConfVars.HIVE_ORC_BLOCK_PADDING_TOLERANCE.varname,
+ HiveConf.ConfVars.HIVE_ORC_BLOCK_PADDING_TOLERANCE.defaultFloatVal);
+
+ long blockSizeValue = HiveConf.getLongVar(conf, HIVE_ORC_DEFAULT_BLOCK_SIZE);
+
+ double bloomFilterFpp = BloomFilterIO.DEFAULT_FPP;
+
+ ObjectInspector inspector = OrcStruct.createObjectInspector(orcSchema);
+
+ return new OrcFlowFileWriter(flowFileOutputStream,
+ path,
+ conf,
+ inspector,
+ stripeSize,
+ compress,
+ bufferSize,
+ rowIndexStride,
+ getMemoryManager(conf),
+ addBlockPadding,
+ versionValue,
+ null, // no callback
+ encodingStrategy,
+ compressionStrategy,
+ paddingTolerance,
+ blockSizeValue,
+ null, // no Bloom Filter column names
+ bloomFilterFpp);
+ }
+
+ private static MemoryManager memoryManager = null;
+
+ private static synchronized MemoryManager getMemoryManager(Configuration conf) {
+ if (memoryManager == null) {
+ memoryManager = new MemoryManager(conf);
+ }
+ return memoryManager;
+ }
+}
\ No newline at end of file