You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2020/05/14 00:06:27 UTC
[asterixdb] 19/20: Merge commit 'f1cd417' from 'gerrit/mad-hatter'
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 54bff8184513edbe511f4e0c73ec6f52b10ee59e
Merge: b564661 f1cd417
Author: Michael Blow <mb...@apache.org>
AuthorDate: Wed May 13 15:19:24 2020 -0400
Merge commit 'f1cd417' from 'gerrit/mad-hatter'
Change-Id: I0f30399a6485708f4b154af09ea36377fe4d3dbf
asterixdb/asterix-app/data/csv/01.csv | 3 +
asterixdb/asterix-app/data/csv/02.csv | 3 +
asterixdb/asterix-app/data/csv/empty.csv | 5 +
asterixdb/asterix-app/data/csv/sample_09.csv | 17 ++
asterixdb/asterix-app/data/csv/sample_10.csv | 39 ++++
asterixdb/asterix-app/data/csv/sample_11.csv | 4 +
asterixdb/asterix-app/data/csv/sample_12.csv | 16 ++
asterixdb/asterix-app/data/csv/sample_13.csv | 11 ++
asterixdb/asterix-app/data/tsv/01.tsv | 3 +
asterixdb/asterix-app/data/tsv/02.tsv | 3 +
asterixdb/asterix-app/data/tsv/sample_01.tsv | 28 +++
.../asterix/app/translator/QueryTranslator.java | 54 +++--
.../aws/AwsS3ExternalDatasetTest.java | 27 +++
.../test/runtime/ResultStreamingFailureTest.java | 83 ++++++++
.../csv-parser-001/csv-parser-001.1.ddl.sqlpp | 35 ++++
.../csv-parser-001/csv-parser-001.2.query.sqlpp} | 8 +-
.../csv-parser-001/csv-parser-001.3.query.sqlpp} | 8 +-
.../csv-parser-001/csv-parser-001.4.query.sqlpp} | 8 +-
.../csv-parser-001/csv-parser-001.5.query.sqlpp} | 8 +-
.../csv-parser-001/csv-parser-001.6.query.sqlpp} | 8 +-
.../csv-parser-001/csv-parser-001.7.query.sqlpp} | 8 +-
.../csv-parser-001/csv-parser-001.8.ddl.sqlpp} | 8 +-
.../tsv-parser-001/tsv-parser-002.1.ddl.sqlpp} | 14 +-
.../tsv-parser-001/tsv-parser-002.2.query.sqlpp} | 8 +-
.../tsv-parser-001/tsv-parser-002.3.ddl.sqlpp} | 8 +-
.../aws/s3/001/query-dataset.000.ddl.sqlpp} | 24 ++-
.../aws/s3/001/query-dataset.002.query.sqlpp} | 8 +-
.../aws/s3/001/query-dataset.003.ddl.sqlpp} | 8 +-
.../aws/s3/002/query-dataset.000.ddl.sqlpp} | 24 ++-
.../aws/s3/002/query-dataset.002.query.sqlpp} | 8 +-
.../aws/s3/002/query-dataset.003.ddl.sqlpp} | 8 +-
.../aws/s3/negative/negative.000.ddl.sqlpp} | 23 ++-
.../get-item_03/get-item_03.4.query.sqlpp} | 19 +-
.../misc/case_01/case_01.1.query.sqlpp | 2 +-
.../misc/case_02/case_02.1.query.sqlpp | 2 +-
.../misc/case_03/case_03.1.query.sqlpp | 2 +-
.../misc/case_04/case_04.1.query.sqlpp | 2 +-
.../misc/case_05/case_05.1.query.sqlpp | 2 +-
.../misc/case_06/case_06.1.query.sqlpp | 2 +-
.../misc/case_07/case_07.1.query.sqlpp | 2 +-
.../numeric/add_double/add_double.1.query.sqlpp | 2 +-
.../union/union_nested/union_nested.1.ddl.sqlpp | 48 +++--
.../union_nested/union_nested.2.update.sqlpp} | 8 +-
.../union/union_nested/union_nested.3.query.sqlpp | 93 +++++++++
.../csv-parser-001/csv-parser-001.2.adm | 15 ++
.../csv-parser-001/csv-parser-001.3.adm | 32 +++
.../csv-parser-001/csv-parser-001.4.adm | 4 +
.../csv-parser-001/csv-parser-001.5.adm | 13 ++
.../csv-parser-001/csv-parser-001.6.adm | 3 +
.../csv-parser-001/csv-parser-001.7.adm | 0
.../tsv-parser-001/tsv-parser-001.2.adm | 28 +++
.../aws/s3/001/external_dataset.001.adm | 6 +
.../aws/s3/002/external_dataset.001.adm | 6 +
.../{get-item_03.1.adm => get-item_03.3.adm} | 0
.../results/list/get-item_03/get-item_03.4.adm | 1 +
.../results/union/union_nested/union_nested.3.adm | 138 +++++++++++++
.../numeric/add_double/add_double.1.ast | 7 +-
.../runtimets/testsuite_external_dataset.xml | 18 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 17 ++
.../asterix/common/exceptions/ErrorCode.java | 6 +-
.../src/main/resources/asx_errormsg/en.properties | 6 +-
.../asterix-doc/src/main/markdown/sqlpp/2_expr.md | 9 +-
.../external/input/HDFSDataSourceFactory.java | 2 +-
.../external/input/record/CharArrayRecord.java | 6 +
.../CSVToRecordWithMetadataAndPKConverter.java | 13 +-
.../record/reader/aws/AwsS3InputStreamFactory.java | 17 +-
.../record/reader/aws/AwsS3ReaderFactory.java | 18 --
.../stream/EmptyLineSeparatedRecordReader.java | 3 +-
.../record/reader/stream/LineRecordReader.java | 28 ++-
.../reader/stream/QuotedLineRecordReader.java | 87 ++++----
.../reader/stream/SemiStructuredRecordReader.java | 4 +-
.../record/reader/stream/StreamRecordReader.java | 3 +-
.../reader/stream/StreamRecordReaderFactory.java | 2 +-
.../input/stream/AsterixInputStreamReader.java | 4 +-
.../input/stream/SocketServerInputStream.java | 5 +
.../external/parser/DelimitedDataParser.java | 65 +++---
.../parser/factory/DelimitedDataParserFactory.java | 52 +----
.../factory/RecordWithMetadataParserFactory.java | 2 +-
.../external/provider/AdapterFactoryProvider.java | 15 +-
.../external/provider/ParserFactoryProvider.java | 7 +-
.../util/ExternalDataCompatibilityUtils.java | 17 +-
.../external/util/ExternalDataConstants.java | 35 +++-
.../asterix/external/util/ExternalDataUtils.java | 220 +++++++++++++++------
.../parser/test/ByteBufUTF8DecodeTest.java | 5 +-
.../external/parser/test/RecordWithMetaTest.java | 5 +-
.../DatasetTupleTranslator.java | 103 ++++++----
.../asterix/metadata/feeds/FeedMetadataUtil.java | 10 +-
.../evaluators/functions/GetItemDescriptor.java | 4 +
asterixdb/asterix-server/pom.xml | 4 +-
.../appended-resources/supplemental-models.xml | 6 +-
....com_netty_netty_netty-4.1.48.Final_NOTICE.txt} | 0
.../apache/hyracks/api/channels/IInputChannel.java | 5 +
.../src/main/resources/errormsg/en.properties | 2 +-
.../hyracks/client/result/ResultSetReader.java | 65 +++---
.../hyracks/comm/channels/NetworkInputChannel.java | 5 +
.../comm/channels/ResultNetworkInputChannel.java | 6 +
.../MaterializedPartitionInputChannel.java | 5 +
.../common/data/parsers/BooleanParserFactory.java | 62 ++++++
.../std/file/DelimitedDataTupleParserFactory.java | 9 +-
.../file/FieldCursorForDelimitedDataParser.java | 179 ++++++++++-------
.../hyracks/dataflow/std/file/CursorTest.java | 5 +-
.../muxdemux/AbstractChannelWriteInterface.java | 10 +-
.../muxdemux/FullFrameChannelWriteInterface.java | 18 +-
.../storage/am/common/impls/AbstractTreeIndex.java | 12 +-
.../storage/common/compression/file/LAFWriter.java | 23 ++-
.../org/apache/hyracks/test/support/TestUtils.java | 5 +
.../java/org/apache/hyracks/util/NetworkUtil.java | 11 ++
.../org/apache/hyracks/util/NetworkUtilTest.java | 42 ++++
hyracks-fullstack/pom.xml | 27 ++-
109 files changed, 1611 insertions(+), 635 deletions(-)
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index f83e21e,3906bd5..ea2ce40
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@@ -681,9 -641,10 +683,10 @@@ public class QueryTranslator extends Ab
case RECORD:
itemTypeDataverseName = dataverseName;
itemTypeName = DatasetUtil.createInlineTypeName(datasetName, false);
- MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName,
- itemTypeDataverseName + "." + itemTypeName);
+ lockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName,
+ itemTypeName);
itemType = translateType(itemTypeDataverseName, itemTypeName, itemTypeExpr, mdTxnCtx);
+ validateDatasetItemType(dsType, itemType, false, sourceLoc);
MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
new Datatype(itemTypeDataverseName, itemTypeName, itemType, true));
break;
@@@ -728,10 -681,11 +723,11 @@@
case RECORD:
metaItemTypeDataverseName = dataverseName;
metaItemTypeName = DatasetUtil.createInlineTypeName(datasetName, true);
- MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(),
- metaItemTypeDataverseName, metaItemTypeDataverseName + "." + metaItemTypeName);
+ lockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(),
+ metaItemTypeDataverseName, metaItemTypeName);
metaItemType = translateType(metaItemTypeDataverseName, metaItemTypeName,
metaItemTypeExpr, mdTxnCtx);
+ validateDatasetItemType(dsType, metaItemType, true, sourceLoc);
MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
new Datatype(metaItemTypeDataverseName, metaItemTypeName, metaItemType, true));
break;
@@@ -2216,11 -1969,15 +2223,13 @@@
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
- MetadataLockUtil.modifyDatasetBegin(lockManager, metadataProvider.getLocks(), dataverseName,
- dataverseName + "." + datasetName);
+ lockUtil.modifyDatasetBegin(lockManager, metadataProvider.getLocks(), dataverseName, datasetName);
try {
- CompiledLoadFromFileStatement cls =
- new CompiledLoadFromFileStatement(dataverseName, loadStmt.getDatasetName(), loadStmt.getAdapter(),
- loadStmt.getProperties(), loadStmt.dataIsAlreadySorted());
+ Map<String, String> properties = loadStmt.getProperties();
+ ExternalDataUtils.normalize(properties);
+ ExternalDataUtils.validate(properties);
- CompiledLoadFromFileStatement cls =
- new CompiledLoadFromFileStatement(dataverseName, loadStmt.getDatasetName().getValue(),
- loadStmt.getAdapter(), properties, loadStmt.dataIsAlreadySorted());
++ CompiledLoadFromFileStatement cls = new CompiledLoadFromFileStatement(dataverseName,
++ loadStmt.getDatasetName(), loadStmt.getAdapter(), properties, loadStmt.dataIsAlreadySorted());
cls.setSourceLocation(stmt.getSourceLocation());
JobSpecification spec = apiFramework.compileQuery(hcc, metadataProvider, null, 0, null, sessionOutput, cls,
null, responsePrinter, warningCollector);
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 7fef69e,14f7c83..4e10498
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@@ -314,7 -307,8 +314,9 @@@ public class ErrorCode
public static final int FAILED_TO_PARSE_METADATA = 3115;
public static final int INPUT_DECODE_FAILURE = 3116;
public static final int FAILED_TO_PARSE_MALFORMED_LOG_RECORD = 3117;
- public static final int METADATA_DROP_LIBRARY_IN_USE = 3118;
+ public static final int PARAMETERS_REQUIRED = 3118;
+ public static final int MALFORMED_RECORD = 3119;
++ public static final int METADATA_DROP_LIBRARY_IN_USE = 3120;
// Lifecycle management errors
public static final int DUPLICATE_PARTITION_ID = 4000;
diff --cc asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 4461ff1,98ae1a7..48d849f
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@@ -312,7 -305,8 +312,9 @@@
3115 = Failed to parse record metadata
3116 = Failed to decode input
3117 = Failed to parse record, malformed log record
- 3118 = Library %1$s is being used. It cannot be dropped
+ 3118 = Parameter(s) %1$s must be specified
+ 3119 = Record number %1$s is malformed
++3120 = Library %1$s is being used. It cannot be dropped
# Lifecycle management errors
4000 = Partition id %1$s for node %2$s already in use by node %3$s
diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 8aebd90,b7b441b..652eae5
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@@ -145,34 -145,17 +149,17 @@@ public class ExternalDataUtils
}
}
- public static String getDataverse(Map<String, String> configuration) {
- return configuration.get(ExternalDataConstants.KEY_DATAVERSE);
+ public static DataverseName getDataverse(Map<String, String> configuration) {
+ return DataverseName.createFromCanonicalForm(configuration.get(ExternalDataConstants.KEY_DATAVERSE));
}
- public static String getRecordFormat(Map<String, String> configuration) {
- String parserFormat = configuration.get(ExternalDataConstants.KEY_DATA_PARSER);
- return parserFormat != null ? parserFormat : configuration.get(ExternalDataConstants.KEY_FORMAT);
- }
-
- public static void setRecordFormat(Map<String, String> configuration, String format) {
- if (!configuration.containsKey(ExternalDataConstants.KEY_DATA_PARSER)) {
- configuration.put(ExternalDataConstants.KEY_DATA_PARSER, format);
- }
- if (!configuration.containsKey(ExternalDataConstants.KEY_FORMAT)) {
- configuration.put(ExternalDataConstants.KEY_FORMAT, format);
+ public static String getParserFactory(Map<String, String> configuration) {
+ String parserFactory = configuration.get(ExternalDataConstants.KEY_PARSER);
+ if (parserFactory != null) {
+ return parserFactory;
}
- }
-
- private static Map<ATypeTag, IValueParserFactory> valueParserFactoryMap = initializeValueParserFactoryMap();
-
- private static Map<ATypeTag, IValueParserFactory> initializeValueParserFactoryMap() {
- Map<ATypeTag, IValueParserFactory> m = new EnumMap<>(ATypeTag.class);
- m.put(ATypeTag.INTEGER, IntegerParserFactory.INSTANCE);
- m.put(ATypeTag.FLOAT, FloatParserFactory.INSTANCE);
- m.put(ATypeTag.DOUBLE, DoubleParserFactory.INSTANCE);
- m.put(ATypeTag.BIGINT, LongParserFactory.INSTANCE);
- m.put(ATypeTag.STRING, UTF8StringParserFactory.INSTANCE);
- return m;
+ parserFactory = configuration.get(ExternalDataConstants.KEY_FORMAT);
+ return parserFactory != null ? parserFactory : configuration.get(ExternalDataConstants.KEY_PARSER_FACTORY);
}
public static IValueParserFactory[] getValueParserFactories(ARecordType recordType) {
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 64b703b,74f5076..f0567a4
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@@ -64,6 -68,9 +65,7 @@@ import org.apache.asterix.om.types.AUno
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.compression.CompressionManager;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+ import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@@ -395,8 -409,39 +390,36 @@@ public class DatasetTupleTranslator ext
return tuple;
}
+ protected void writeCompactionPolicy(DatasetType datasetType, String compactionPolicy,
+ Map<String, String> compactionPolicyProperties, OrderedListBuilder listBuilder,
+ ArrayBackedValueStorage itemValue) throws HyracksDataException {
+ // write field 6
+ fieldValue.reset();
+ aString.setValue(compactionPolicy);
+ stringSerde.serialize(aString, fieldValue.getDataOutput());
+ recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_FIELD_INDEX, fieldValue);
+
+ // write field 7
+ listBuilder.reset((AOrderedListType) MetadataRecordTypes.DATASET_RECORDTYPE
+ .getFieldTypes()[MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX]);
+ if (compactionPolicyProperties != null && !compactionPolicyProperties.isEmpty()) {
+ for (Map.Entry<String, String> property : compactionPolicyProperties.entrySet()) {
+ String name = property.getKey();
+ String value = property.getValue();
+ itemValue.reset();
+ DatasetUtil.writePropertyTypeRecord(name, value, itemValue.getDataOutput(),
+ MetadataRecordTypes.COMPACTION_POLICY_PROPERTIES_RECORDTYPE);
+ listBuilder.addItem(itemValue);
+ }
+ }
+ fieldValue.reset();
+ listBuilder.write(fieldValue.getDataOutput(), true);
+ recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX,
+ fieldValue);
+ }
+
/**
* Keep protected to allow other extensions to add additional fields
- *
- * @param dataset
- * @throws HyracksDataException
*/
protected void writeOpenFields(Dataset dataset) throws HyracksDataException {
writeMetaPart(dataset);