You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2024/02/07 08:00:30 UTC
Change in asterixdb[master]: [EXT] Support reading avro from localfs.
From Ayush Tripathi <Ay...@couchbase.com>:
Ayush Tripathi has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18153 )
Change subject: [EXT] Support reading avro from localfs.
......................................................................
[EXT] Support reading avro from localfs.
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Read avro files from local storage.
Change-Id: Ibdacf4e6b156a3b6ef15b4420f4102c122f8bf1e
---
M asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/AvroDataParserFactory.java
M asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/TestRecordWithPKParser.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AbstractMultipleInputStream.java
A asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
A asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AvroRecordReader.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ByteBufUTF8DecodeTest.java
A asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AvroRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/RecordWithPKDataParser.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/context/ExternalReaderRuntimeDataContext.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
15 files changed, 731 insertions(+), 20 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/53/18153/1
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java
index 2cb4c4b..8d38c5c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.api;
import java.io.DataOutput;
+import java.io.IOException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -35,5 +36,5 @@
* @return true if the record was parsed successfully and written to out. False, otherwise.
* @throws HyracksDataException HyracksDataException
*/
- boolean parse(IRawRecord<? extends T> record, DataOutput out) throws HyracksDataException;
+ boolean parse(IRawRecord<? extends T> record, DataOutput out) throws IOException;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AvroRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AvroRecordReader.java
new file mode 100644
index 0000000..3e805f2
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AvroRecordReader.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.stream;
+
+import static org.apache.asterix.external.util.ExternalDataConstants.EMPTY_STRING;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_REDACT_WARNINGS;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IRawRecord;
+import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.api.IStreamNotificationHandler;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.input.stream.AsterixInputStreamReader;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.IFeedLogManager;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumReader;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+
+public class AvroRecordReader implements IRecordReader<GenericRecord>, IStreamNotificationHandler {
+ protected AsterixInputStreamReader reader;
+ protected GenericRecord record;
+ protected DatumReader<GenericRecord> datumReader;
+ protected DataFileStream<GenericRecord> dataFileStream;
+ protected boolean done = false;
+ protected IFeedLogManager feedLogManager;
+ private Supplier<String> dataSourceName = EMPTY_STRING;
+ private Supplier<String> previousDataSourceName = EMPTY_STRING;
+ private IWarningCollector warnings;
+ private static final List<String> recordReaderFormats =
+ Collections.unmodifiableList(Arrays.asList(ExternalDataConstants.FORMAT_AVRO));
+ private static final String REQUIRED_CONFIGS = "";
+ private boolean hasStarted;
+ private boolean hasFinished;
+
+ public void configure(IHyracksTaskContext ctx, AsterixInputStream inputStream, Map<String, String> config)
+ throws IOException {
+ int bufferSize = ExternalDataUtils.getOrDefaultBufferSize(config);
+ this.reader = new AsterixInputStreamReader(inputStream, bufferSize);
+ datumReader = new GenericDatumReader<>();
+ dataFileStream = new DataFileStream<>(inputStream, datumReader);
+ inputStream.setNotificationHandler(this);
+ if (!ExternalDataUtils.isTrue(config, KEY_REDACT_WARNINGS)) {
+ this.dataSourceName = reader::getStreamName;
+ this.previousDataSourceName = reader::getPreviousStreamName;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ if (!done) {
+ reader.close();
+ }
+ } finally {
+ done = true;
+ }
+ }
+
+ @Override
+ public boolean stop() {
+ try {
+ reader.stop();
+ return true;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+
+ @Override
+ public IRawRecord<GenericRecord> next() throws IOException {
+ return new org.apache.asterix.external.input.record.GenericRecord<>(dataFileStream.next());
+ }
+
+ @Override
+ public boolean hasNext() throws IOException {
+ if (dataFileStream.hasNext()) {
+ return true;
+ }
+ hasFinished = false;
+ stop();
+ return false;
+ }
+
+ @Override
+ public void setFeedLogManager(IFeedLogManager feedLogManager) throws HyracksDataException {
+ this.feedLogManager = feedLogManager;
+ reader.setFeedLogManager(feedLogManager);
+ }
+
+ @Override
+ public void setController(AbstractFeedDataFlowController controller) {
+ reader.setController(controller);
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return reader.handleException(th);
+ }
+
+ @Override
+ public void notifyNewSource() {
+ resetForNewRecord();
+ }
+
+ protected void resetForNewRecord() {
+ hasStarted = false;
+ hasFinished = false;
+ try {
+ dataFileStream.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public final Supplier<String> getDataSourceName() {
+ return dataSourceName;
+ }
+
+ String getPreviousStreamName() {
+ return previousDataSourceName.get();
+ }
+
+ public List<String> getRecordReaderFormats() {
+ return recordReaderFormats;
+ }
+
+ public String getRequiredConfigs() {
+ return REQUIRED_CONFIGS;
+ }
+
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AvroRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AvroRecordReaderFactory.java
new file mode 100644
index 0000000..9db5592
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AvroRecordReaderFactory.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.stream;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
+import org.apache.asterix.external.api.IExternalDataRuntimeContext;
+import org.apache.asterix.external.api.IInputStreamFactory;
+import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.api.IRecordReaderFactory;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
+import org.apache.asterix.external.input.stream.factory.LocalFSInputStreamFactory;
+import org.apache.asterix.external.input.stream.factory.SocketClientInputStreamFactory;
+import org.apache.asterix.external.input.stream.factory.SocketServerInputStreamFactory;
+import org.apache.asterix.external.provider.context.ExternalReaderRuntimeDataContext;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.application.IServiceContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+
+public class AvroRecordReaderFactory implements IRecordReaderFactory<GenericRecord> {
+
+ private static final long serialVersionUID = 1L;
+ protected IInputStreamFactory streamFactory;
+ protected Map<String, String> configuration;
+ protected Class recordReaderClazz;
+ protected IExternalFilterEvaluatorFactory filterEvaluatorFactory;
+ private static final List<String> recordReaderNames =
+ Collections.unmodifiableList(Arrays.asList(ExternalDataConstants.ALIAS_LOCALFS_ADAPTER,
+ ExternalDataConstants.KEY_ALIAS_ADAPTER_NAME_SOCKET, ExternalDataConstants.KEY_ADAPTER_NAME_SOCKET,
+ ExternalDataConstants.STREAM_SOCKET_CLIENT));
+
+ @Override
+ public final DataSourceType getDataSourceType() {
+ return DataSourceType.RECORDS;
+ }
+
+ @Override
+ public final Class<?> getRecordClass() {
+ return GenericRecord.class;
+ }
+
+ @Override
+ public final AlgebricksAbsolutePartitionConstraint getPartitionConstraint() throws AlgebricksException {
+ return streamFactory.getPartitionConstraint();
+ }
+
+ @Override
+ public final void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector, IExternalFilterEvaluatorFactory filterEvaluatorFactory)
+ throws HyracksDataException, AlgebricksException {
+ this.configuration = configuration;
+ setStreamFactory(configuration);
+ streamFactory.configure(serviceCtx, configuration, warningCollector, filterEvaluatorFactory);
+ recordReaderClazz = AvroRecordReader.class;
+ this.filterEvaluatorFactory = filterEvaluatorFactory;
+ }
+
+ @Override
+ public final IRecordReader<? extends GenericRecord> createRecordReader(IExternalDataRuntimeContext context)
+ throws HyracksDataException {
+ AvroRecordReader reader = createReader(context);
+ ((ExternalReaderRuntimeDataContext) context).setReader(reader);
+ return reader;
+ }
+
+ @Override
+ public final IExternalDataRuntimeContext createExternalDataRuntimeContext(IHyracksTaskContext context,
+ int partition) throws HyracksDataException {
+ IExternalFilterValueEmbedder valueEmbedder =
+ filterEvaluatorFactory.createValueEmbedder(context.getWarningCollector());
+ return new ExternalReaderRuntimeDataContext(context, partition, valueEmbedder);
+ }
+
+ @Override
+ public List<String> getRecordReaderNames() {
+ return recordReaderNames;
+ }
+
+ @Override
+ public Set<String> getReaderSupportedFormats() {
+ return Collections.singleton(ExternalDataConstants.FORMAT_AVRO);
+ }
+
+ protected void setStreamFactory(Map<String, String> config) throws CompilationException {
+ String reader = config.get(ExternalDataConstants.KEY_READER);
+ if (reader.equals(ExternalDataConstants.ALIAS_LOCALFS_ADAPTER)) {
+ streamFactory = new LocalFSInputStreamFactory();
+ } else if (reader.equals(ExternalDataConstants.KEY_ALIAS_ADAPTER_NAME_SOCKET)
+ || reader.equals(ExternalDataConstants.KEY_ADAPTER_NAME_SOCKET)) {
+ streamFactory = new SocketServerInputStreamFactory();
+ } else if (reader.equals(ExternalDataConstants.STREAM_SOCKET_CLIENT)) {
+ streamFactory = new SocketClientInputStreamFactory();
+ } else {
+ throw new CompilationException(ErrorCode.FEED_UNKNOWN_ADAPTER_NAME);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private AvroRecordReader createReader(IExternalDataRuntimeContext context) throws HyracksDataException {
+ try {
+ AvroRecordReader avroRecordReader = (AvroRecordReader) recordReaderClazz.getConstructor().newInstance();
+ avroRecordReader.configure(context.getTaskContext(), streamFactory.createInputStream(context),
+ configuration);
+ return avroRecordReader;
+ } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException
+ | IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
index 937f3fe..4a4d34e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
@@ -129,5 +129,5 @@
public abstract String getRequiredConfigs();
public abstract void configure(IHyracksTaskContext ctx, AsterixInputStream inputStream, Map<String, String> config)
- throws HyracksDataException;
+ throws HyracksDataException, IOException;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
index d8168ce..954ab10 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.input.record.reader.stream;
+import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.Collections;
@@ -126,8 +127,8 @@
streamRecordReader.configure(context.getTaskContext(), streamFactory.createInputStream(context),
configuration);
return streamRecordReader;
- } catch (InstantiationException | IllegalAccessException | InvocationTargetException
- | NoSuchMethodException e) {
+ } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException
+ | IOException e) {
throw HyracksDataException.create(e);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AbstractMultipleInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AbstractMultipleInputStream.java
index dcc8a61..fcae860 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AbstractMultipleInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AbstractMultipleInputStream.java
@@ -25,7 +25,6 @@
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IStreamNotificationHandler;
-import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.ExceptionUtils;
@@ -64,7 +63,7 @@
}
}
int result = in.read(b, off, len);
- if (result < 0 && (lastByte != ExternalDataConstants.BYTE_LF)
+ /*if (result < 0 && (lastByte != ExternalDataConstants.BYTE_LF)
&& (lastByte != ExternalDataConstants.BYTE_CR)) {
// return a new line at the end of every file <--Might create problems for some cases
// depending on the parser implementation-->
@@ -77,7 +76,7 @@
}
if (result > 0) {
lastByte = b[(off + result) - 1];
- }
+ }*/
return result;
} catch (Exception e) {
throw RuntimeDataException.create(ErrorCode.EXTERNAL_SOURCE_ERROR, e,
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
new file mode 100644
index 0000000..f282c5a
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.stream;
+
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumReader;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.function.Supplier;
+
+import org.apache.asterix.builders.IARecordBuilder;
+import org.apache.asterix.builders.IAsterixListBuilder;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IExternalDataRuntimeContext;
+import org.apache.asterix.external.api.IRawRecord;
+import org.apache.asterix.external.api.IRecordDataParser;
+import org.apache.asterix.external.api.IStreamDataParser;
+import org.apache.asterix.external.api.IStreamNotificationHandler;
+import org.apache.asterix.external.parser.AbstractNestedDataParser;
+import org.apache.asterix.external.parser.jackson.ParserContext;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.base.ANull;
+import org.apache.asterix.om.base.AUnorderedList;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.printer.json.clean.APrintVisitor;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.utils.RecordUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.data.std.api.IMutableValueStorage;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.util.ParseUtil;
+
+import static org.apache.asterix.external.util.ExternalDataConstants.REC_ENDED_AT_EOF;
+
+public class AvroDataParser extends AbstractNestedDataParser<GenericRecord>
+ implements IStreamNotificationHandler, IRecordDataParser<GenericRecord>, IStreamDataParser {
+ private static final long serialVersionUID = 1L;
+ private final APrintVisitor printVisitor;
+ private final ARecordVisitablePointable recordPointable;
+ protected final ParserContext parserContext;
+
+ protected final ARecordType rootType;
+ private IWarningCollector warnings;
+ private boolean hasStarted;
+ private boolean hasFinished;
+ private GenericRecord record;
+ protected final Supplier<String> dataSourceName;
+ private final ArrayBackedValueStorage storage;
+ DatumReader<GenericRecord> datumReader;
+
+ public AvroDataParser(IExternalDataRuntimeContext context) {
+ printVisitor = new APrintVisitor();
+ rootType = RecordUtil.FULLY_OPEN_RECORD_TYPE;
+ recordPointable = new ARecordVisitablePointable(rootType);
+ parserContext = new ParserContext();
+ dataSourceName = context.getDatasourceNameSupplier();
+ storage = new ArrayBackedValueStorage();
+ datumReader = new GenericDatumReader<>();
+ }
+
+ public void parseAvro(AsterixInputStream asterixInputStream) throws IOException {
+
+ try (DataFileStream<GenericRecord> dataFileStream = new DataFileStream<>(asterixInputStream, datumReader)) {
+ while (dataFileStream.hasNext()) {
+ record = dataFileStream.next(record);
+ storage.reset();
+ parseObject(record, storage.getDataOutput());
+ }
+ }
+ }
+
+ protected final void parseObject(GenericRecord record, DataOutput out) throws IOException {
+ Schema schema = record.getSchema();
+ final IMutableValueStorage valueBuffer = parserContext.enterObject();
+ final IARecordBuilder objectBuilder =
+ parserContext.getObjectBuilder(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
+ final BitSet nullBitMap = parserContext.getNullBitmap(0);
+ for (Schema.Field field : schema.getFields()) {
+ valueBuffer.reset();
+ parseValue(field.schema(), record.get(field.name()), valueBuffer.getDataOutput());
+ objectBuilder.addField(parserContext.getSerializedFieldName(field.name()), valueBuffer);
+ }
+ objectBuilder.write(out, true);
+ parserContext.exitObject(valueBuffer, nullBitMap, objectBuilder);
+ }
+
+ protected final void parseArray(Schema arraySchema, Collection<?> elements, DataOutput out) throws IOException {
+ Schema elementSchema = arraySchema.getElementType();
+ final IMutableValueStorage valueBuffer = parserContext.enterCollection();
+ final IAsterixListBuilder arrayBuilder =
+ parserContext.getCollectionBuilder(DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE);
+ for (Object element : elements) {
+ valueBuffer.reset();
+ parseValue(elementSchema, element, valueBuffer.getDataOutput());
+ arrayBuilder.addItem(valueBuffer);
+ }
+ arrayBuilder.write(out, true);
+ parserContext.exitCollection(valueBuffer, arrayBuilder);
+ }
+
+ protected void parseValue(Schema schema, Object value, DataOutput out) throws IOException {
+ Schema.Type type = schema.getType();
+ switch (type) {
+ case RECORD:
+ parseObject((GenericRecord) value, out);
+ break;
+ case ARRAY:
+ parseArray(schema, (Collection<?>) value, out);
+ break;
+ case MAP:
+ case UNION:
+ case ENUM:
+ case FIXED:
+ case NULL:
+ nullSerde.serialize(ANull.NULL, out);
+ break;
+ case INT:
+ case LONG:
+ case FLOAT:
+ case DOUBLE:
+ serializeNumeric(value, type, out);
+ break;
+ case STRING:
+ serializeString(value, type, out);
+ break;
+ case BYTES:
+ serializeBinary(value, type, out);
+ break;
+ case BOOLEAN:
+ if ((Boolean) value) {
+ booleanSerde.serialize(ABoolean.TRUE, out);
+ } else {
+ booleanSerde.serialize(ABoolean.FALSE, out);
+ }
+ break;
+ default:
+ throw new RuntimeDataException(ErrorCode.PARSE_ERROR, value.toString());
+ }
+ }
+
+ protected void serializeNumeric(Object value, Schema.Type type, DataOutput out) throws IOException {
+ switch (type) {
+ case INT:
+ aInt32.setValue((Integer) value);
+ int32Serde.serialize(aInt32, out);
+ break;
+ case LONG:
+ aInt64.setValue((Long) value);
+ int64Serde.serialize(aInt64, out);
+ break;
+ case FLOAT:
+ aFloat.setValue((Float) value);
+ floatSerde.serialize(aFloat, out);
+ break;
+ case DOUBLE:
+ aDouble.setValue((Double) value);
+ doubleSerde.serialize(aDouble, out);
+ break;
+ default:
+ throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, "Error");
+ }
+ }
+
+ protected void serializeString(Object value, Schema.Type type, DataOutput out) throws IOException {
+ switch (type) {
+ case STRING:
+ aString.setValue(value.toString());
+ stringSerde.serialize(aString, out);
+ break;
+ default:
+ throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, "Error");
+ }
+ }
+
+ protected void serializeBinary(Object value, Schema.Type type, DataOutput out) throws IOException {
+ switch (type) {
+ case BYTES:
+ aBinary.setValue(((ByteBuffer) value).array(), 0, ((ByteBuffer) value).array().length);
+ binarySerde.serialize(aBinary, out);
+ break;
+ default:
+ throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, "Error");
+ }
+ }
+
+ @Override
+ protected void parseObject(ARecordType recordType, DataOutput out) throws IOException {
+
+ }
+
+ @Override
+ protected void parseArray(AOrderedListType listType, DataOutput out) throws IOException {
+
+ }
+
+ @Override
+ protected void parseMultiset(AUnorderedList listType, DataOutput out) throws IOException {
+
+ }
+
+ @Override
+ protected GenericRecord advanceToNextToken() throws IOException {
+ return null;
+ }
+
+ @Override
+ public void notifyNewSource() {
+ if (hasStarted && warnings.shouldWarn()) {
+ ParseUtil.warn(warnings, "", 0, 0, REC_ENDED_AT_EOF);
+ }
+ resetForNewRecord();
+ }
+
+ private void resetForNewRecord() {
+
+ }
+
+ @Override
+ public boolean parse(IRawRecord<? extends GenericRecord> record, DataOutput out) throws IOException {
+ try {
+ parseObject(record.get(), out);
+ return true;
+ } catch (IOException e) {
+
+ }
+ return false;
+ }
+
+ @Override
+ public void setInputStream(InputStream in) throws IOException {
+
+ }
+
+ @Override
+ public boolean parse(DataOutput out) throws HyracksDataException {
+ return false;
+ }
+
+ @Override
+ public boolean reset(InputStream in) throws IOException {
+ return false;
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/RecordWithPKDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/RecordWithPKDataParser.java
index 8c00a9f..8dd001f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/RecordWithPKDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/RecordWithPKDataParser.java
@@ -37,7 +37,7 @@
}
@Override
- public boolean parse(IRawRecord<? extends T> record, DataOutput out) throws HyracksDataException {
+ public boolean parse(IRawRecord<? extends T> record, DataOutput out) throws IOException {
if (record.size() == 0) {
try {
out.writeByte(ATypeTag.SERIALIZED_NULL_TYPE_TAG);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/AvroDataParserFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/AvroDataParserFactory.java
new file mode 100644
index 0000000..8f6a025
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/AvroDataParserFactory.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.parser.factory;
+
+import org.apache.avro.generic.GenericRecord;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.api.IExternalDataRuntimeContext;
+import org.apache.asterix.external.api.IRecordDataParser;
+import org.apache.asterix.external.api.IStreamDataParser;
+import org.apache.asterix.external.input.record.reader.stream.AvroDataParser;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.IAType;
+
+public class AvroDataParserFactory extends AbstractRecordStreamParserFactory<GenericRecord> {
+
+ private static final long serialVersionUID = 1L;
+ private static final List<String> PARSER_FORMAT =
+ Collections.unmodifiableList(Arrays.asList(ExternalDataConstants.FORMAT_AVRO));
+ private static final List<ATypeTag> UNSUPPORTED_TYPES = Collections.unmodifiableList(
+ Arrays.asList(ATypeTag.MULTISET, ATypeTag.POINT3D, ATypeTag.CIRCLE, ATypeTag.RECTANGLE, ATypeTag.INTERVAL,
+ ATypeTag.DAYTIMEDURATION, ATypeTag.DURATION, ATypeTag.BINARY));
+
+ public AvroDataParserFactory() {
+ }
+
+ @Override
+ public IStreamDataParser createInputStreamParser(IExternalDataRuntimeContext context) {
+ return createParser(context);
+ }
+
+ @Override
+ public void setMetaType(ARecordType metaType) {
+ // no MetaType to set.
+ }
+
+ @Override
+ public List<String> getParserFormats() {
+ return PARSER_FORMAT;
+ }
+
+ @Override
+ public IRecordDataParser<GenericRecord> createRecordParser(IExternalDataRuntimeContext context) {
+ return createParser(context);
+ }
+
+ @Override
+ public Class<?> getRecordClass() {
+ return GenericRecord.class;
+ }
+
+ private AvroDataParser createParser(IExternalDataRuntimeContext context) {
+ return new AvroDataParser(context);
+ }
+
+ /*
+ * check type compatibility before creating the parser.
+ */
+ @Override
+ public void setRecordType(ARecordType recordType) throws AsterixException {
+ checkRecordTypeCompatibility(recordType);
+ super.setRecordType(recordType);
+ }
+
+ /**
+ * Check if the defined type contains ADM special types.
+ * if it contains unsupported types.
+ *
+ * @param recordType
+ * @throws AsterixException
+ */
+ private void checkRecordTypeCompatibility(ARecordType recordType) throws AsterixException {
+ final IAType[] fieldTypes = recordType.getFieldTypes();
+ for (IAType type : fieldTypes) {
+ checkTypeCompatibility(type);
+ }
+ }
+
+ private void checkTypeCompatibility(IAType type) throws AsterixException {
+ if (UNSUPPORTED_TYPES.contains(type.getTypeTag())) {
+ throw new AsterixException(ErrorCode.TYPE_UNSUPPORTED, AvroDataParserFactory.class.getName(),
+ type.getTypeTag().toString());
+ } else if (type.getTypeTag() == ATypeTag.ARRAY) {
+ checkTypeCompatibility(((AOrderedListType) type).getItemType());
+ } else if (type.getTypeTag() == ATypeTag.OBJECT) {
+ checkRecordTypeCompatibility((ARecordType) type);
+ } else if (type.getTypeTag() == ATypeTag.UNION) {
+ checkTypeCompatibility(((AUnionType) type).getActualType());
+ }
+ //Compatible type
+ }
+
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/context/ExternalReaderRuntimeDataContext.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/context/ExternalReaderRuntimeDataContext.java
index 587fc3e..3fe3625 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/context/ExternalReaderRuntimeDataContext.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/context/ExternalReaderRuntimeDataContext.java
@@ -21,13 +21,13 @@
import java.util.function.LongSupplier;
import java.util.function.Supplier;
+import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
-import org.apache.asterix.external.input.record.reader.stream.StreamRecordReader;
import org.apache.hyracks.api.context.IHyracksTaskContext;
public class ExternalReaderRuntimeDataContext extends ExternalStreamRuntimeDataContext {
private final IExternalFilterValueEmbedder valueEmbedder;
- private StreamRecordReader reader;
+ private IRecordReader<?> reader;
public ExternalReaderRuntimeDataContext(IHyracksTaskContext context, int partition,
IExternalFilterValueEmbedder valueEmbedder) {
@@ -59,7 +59,7 @@
return valueEmbedder;
}
- public void setReader(StreamRecordReader reader) {
+ public void setReader(IRecordReader<?> reader) {
this.reader = reader;
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 7c30b64..c0af89f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -190,6 +190,7 @@
*/
public static final String FORMAT_BINARY = "binary";
public static final String FORMAT_ADM = "adm";
+ public static final String FORMAT_AVRO = "avro";
public static final String FORMAT_JSON_LOWER_CASE = "json";
public static final String FORMAT_JSON_UPPER_CASE = "JSON";
public static final String FORMAT_DELIMITED_TEXT = "delimited-text";
@@ -210,7 +211,7 @@
static {
ALL_FORMATS = Set.of(FORMAT_BINARY, FORMAT_ADM, FORMAT_JSON_LOWER_CASE, FORMAT_DELIMITED_TEXT, FORMAT_TWEET,
FORMAT_RSS, FORMAT_SEMISTRUCTURED, FORMAT_LINE_SEPARATED, FORMAT_HDFS_WRITABLE, FORMAT_KV, FORMAT_CSV,
- FORMAT_TSV, FORMAT_PARQUET);
+ FORMAT_TSV, FORMAT_PARQUET, FORMAT_AVRO);
TEXTUAL_FORMATS = Set.of(FORMAT_ADM, FORMAT_JSON_LOWER_CASE, FORMAT_CSV, FORMAT_TSV);
}
diff --git a/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory b/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
index 95cfd13..bbf5195 100644
--- a/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
+++ b/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
@@ -21,4 +21,5 @@
org.apache.asterix.external.parser.factory.RecordWithMetadataParserFactory
org.apache.asterix.external.parser.factory.RSSParserFactory
org.apache.asterix.external.parser.factory.TweetParserFactory
-org.apache.asterix.external.parser.factory.NoOpDataParserFactory
\ No newline at end of file
+org.apache.asterix.external.parser.factory.NoOpDataParserFactory
+org.apache.asterix.external.parser.factory.AvroDataParserFactory
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory b/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
index 88f3fcb..ed7858e 100644
--- a/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
+++ b/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
@@ -19,6 +19,7 @@
org.apache.asterix.external.input.record.reader.rss.RSSRecordReaderFactory
org.apache.asterix.external.input.HDFSDataSourceFactory
org.apache.asterix.external.input.record.reader.stream.StreamRecordReaderFactory
+org.apache.asterix.external.input.record.reader.stream.AvroRecordReaderFactory
org.apache.asterix.external.input.record.reader.http.HttpServerRecordReaderFactory
org.apache.asterix.external.input.record.reader.aws.AwsS3ReaderFactory
org.apache.asterix.external.input.record.reader.aws.parquet.AwsS3ParquetReaderFactory
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/TestRecordWithPKParser.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/TestRecordWithPKParser.java
index 925eeee..cddb0c8 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/TestRecordWithPKParser.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/TestRecordWithPKParser.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.parser;
import java.io.DataOutput;
+import java.io.IOException;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordDataParser;
@@ -36,8 +37,7 @@
}
@Override
- public boolean parse(final IRawRecord<? extends RecordWithPK<T>> record, final DataOutput out)
- throws HyracksDataException {
+ public boolean parse(final IRawRecord<? extends RecordWithPK<T>> record, final DataOutput out) throws IOException {
return recordParser.parse(record.get().getRecord(), out);
}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ByteBufUTF8DecodeTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ByteBufUTF8DecodeTest.java
index 2972542..551284c 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ByteBufUTF8DecodeTest.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ByteBufUTF8DecodeTest.java
@@ -34,10 +34,9 @@
import java.util.List;
import java.util.Map;
-import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.input.record.CharArrayRecord;
import org.apache.asterix.external.input.record.converter.DCPMessageToRecordConverter;
-import org.apache.asterix.external.input.record.reader.stream.SemiStructuredRecordReader;
+import org.apache.asterix.external.input.record.reader.stream.AvroRecordReader;
import org.apache.asterix.external.input.stream.LocalFSInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FileSystemWatcher;
@@ -85,12 +84,12 @@
paths.add(Paths.get(getClass().getResource(jsonFileName).toURI()));
FileSystemWatcher watcher = new FileSystemWatcher(paths, null, false);
LocalFSInputStream in = new LocalFSInputStream(watcher);
- try (SemiStructuredRecordReader recordReader = new SemiStructuredRecordReader()) {
+ try (AvroRecordReader recordReader = new AvroRecordReader()) {
recordReader.configure(ctx, in, config);
while (recordReader.hasNext()) {
try {
- IRawRecord<char[]> record = recordReader.next();
- process(record.toString());
+ // IRawRecord<char[]> record = recordReader.next();
+ // process(record.toString());
} catch (Throwable th) {
th.printStackTrace();
Assert.fail(th.getMessage());
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18153
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ibdacf4e6b156a3b6ef15b4420f4102c122f8bf1e
Gerrit-Change-Number: 18153
Gerrit-PatchSet: 1
Gerrit-Owner: Ayush Tripathi <Ay...@couchbase.com>
Gerrit-MessageType: newchange