You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by al...@apache.org on 2023/03/18 02:54:35 UTC
[asterixdb] 01/11: [ASTERIXDB-3122][FUN] escape string values in DUMP_INDEX
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b59f647bff727e4495f50c93ccd396c6d45ce38f
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Tue Feb 28 20:16:26 2023 -0800
[ASTERIXDB-3122][FUN] escape string values in DUMP_INDEX
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: I9127d2346541f2cb2bc9d64016f4608ec4ec695c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17405
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../apache/asterix/app/function/DumpIndexDatasource.java | 7 +++++++
.../org/apache/asterix/app/function/DumpIndexReader.java | 4 ++++
.../misc/dump_index/dump_index.2.update.sqlpp | 3 ++-
.../runtimets/results/misc/dump_index/dump_index.10.adm | 2 +-
.../external/adapter/factory/GenericAdapterFactory.java | 16 +++++++---------
.../asterix/metadata/declared/FunctionDataSource.java | 11 ++++++++++-
6 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 3351075143..691be4706c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -19,6 +19,8 @@
package org.apache.asterix.app.function;
import org.apache.asterix.common.cluster.IClusterStateManager;
+import org.apache.asterix.external.api.IDataParserFactory;
+import org.apache.asterix.external.parser.factory.JSONDataParserFactory;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -59,4 +61,9 @@ public class DumpIndexDatasource extends FunctionDataSource {
AlgebricksAbsolutePartitionConstraint locations) {
return new DumpIndexFunction(locations, indexDataflowHelperFactory, recDesc, comparatorFactories);
}
+
+ @Override
+ protected IDataParserFactory createDataParserFactory() {
+ return new JSONDataParserFactory();
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexReader.java
index e60b015b06..8ef094e923 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexReader.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexReader.java
@@ -25,6 +25,7 @@ import java.nio.ByteBuffer;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.input.record.CharArrayRecord;
import org.apache.asterix.om.base.ARecord;
+import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.base.IACollection;
import org.apache.asterix.om.base.IACursor;
import org.apache.asterix.om.base.IAObject;
@@ -132,6 +133,9 @@ public class DumpIndexReader extends FunctionReader {
case DATETIME:
JSONUtil.quoteAndEscape(recordBuilder, field.toString());
break;
+ case STRING:
+ JSONUtil.quoteAndEscape(recordBuilder, ((AString) field).getStringValue());
+ break;
case MISSING:
break;
default:
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dump_index/dump_index.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dump_index/dump_index.2.update.sqlpp
index 10378f2d82..3469c9a0ac 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dump_index/dump_index.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dump_index/dump_index.2.update.sqlpp
@@ -18,5 +18,6 @@
*/
USE test;
INSERT INTO ds ([{"id":1, "name": "name1"}, {"id":2, "name": "name2"}]);
-UPSERT INTO ds2 ([{"id": 1,"age":30, "a1": {"b": [{"x": [1,2]}, {"x": [1,2]}]}, "a2": [{"x": [1,2]}, {"x": [1,2]}] }]);
+UPSERT INTO ds2 ([{"id": 1,"age":"t\\'s", "a1": {"b": [{"x": [1,2]}, {"x": [1,2]}]}, "a2": [{"x": [1,2]}, {"x": [1,2]}]
+}]);
ANALYZE DATASET ds2;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dump_index/dump_index.10.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dump_index/dump_index.10.adm
index a52ceaeb97..d13e4524a7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dump_index/dump_index.10.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dump_index/dump_index.10.adm
@@ -1 +1 @@
-[ { "values": [ 1, { "id": 1, "age": 30, "a1": { "b": [ { "x": [ 1, 2 ] }, { "x": [ 1, 2 ] } ] }, "a2": [ { "x": [ 1, 2 ] }, { "x": [ 1, 2 ] } ] } ] } ]
\ No newline at end of file
+[ { "values": [ 1, { "id": 1, "age": "t\\'s", "a1": { "b": [ { "x": [ 1, 2 ] }, { "x": [ 1, 2 ] } ] }, "a2": [ { "x": [ 1, 2 ] }, { "x": [ 1, 2 ] } ] } ] } ]
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index ac45f7a72b..8ec5af037e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -37,7 +37,6 @@ import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.dataset.adapter.FeedAdapter;
import org.apache.asterix.external.dataset.adapter.GenericAdapter;
import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.external.parser.factory.ADMDataParserFactory;
import org.apache.asterix.external.provider.DataflowControllerProvider;
import org.apache.asterix.external.provider.DatasourceFactoryProvider;
import org.apache.asterix.external.provider.ParserFactoryProvider;
@@ -47,7 +46,6 @@ import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.asterix.external.util.FeedUtils;
import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.utils.RecordUtil;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.ICCServiceContext;
@@ -209,18 +207,18 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda
}
/**
- * Use pre-configured datasource factory
- * For function datasources
+ * Use pre-configured datasource factory For function datasources
*
* @param dataSourceFactory
- * the function datasource factory
+ * the function datasource factory
+ * @param dataParserFactory
+ * the function data parser factory
* @throws AlgebricksException
*/
- public void configure(IExternalDataSourceFactory dataSourceFactory) throws AlgebricksException {
+ public void configure(IExternalDataSourceFactory dataSourceFactory, IDataParserFactory dataParserFactory)
+ throws AlgebricksException {
this.dataSourceFactory = dataSourceFactory;
- dataParserFactory = new ADMDataParserFactory();
- dataParserFactory.setRecordType(RecordUtil.FULLY_OPEN_RECORD_TYPE);
- dataParserFactory.configure(Collections.emptyMap());
+ this.dataParserFactory = dataParserFactory;
configuration = Collections.emptyMap();
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index a7d560c70c..5c874d7feb 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@ -27,6 +27,8 @@ import java.util.Set;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.external.adapter.factory.GenericAdapterFactory;
+import org.apache.asterix.external.api.IDataParserFactory;
+import org.apache.asterix.external.parser.factory.ADMDataParserFactory;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.RecordUtil;
@@ -98,7 +100,10 @@ public abstract class FunctionDataSource extends DataSource {
IClusterStateManager csm = metadataProvider.getApplicationContext().getClusterStateManager();
FunctionDataSourceFactory factory =
new FunctionDataSourceFactory(createFunction(metadataProvider, getLocations(csm)));
- adapterFactory.configure(factory);
+ IDataParserFactory dataParserFactory = createDataParserFactory();
+ dataParserFactory.setRecordType(RecordUtil.FULLY_OPEN_RECORD_TYPE);
+ dataParserFactory.configure(Collections.emptyMap());
+ adapterFactory.configure(factory, dataParserFactory);
return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
tupleFilterFactory, outputLimit);
}
@@ -112,6 +117,10 @@ public abstract class FunctionDataSource extends DataSource {
return new AlgebricksAbsolutePartitionConstraint(ncs.toArray(new String[ncs.size()]));
}
+ protected IDataParserFactory createDataParserFactory() {
+ return new ADMDataParserFactory();
+ }
+
protected static DataSourceId createDataSourceId(FunctionIdentifier fid, String... parameters) {
return new DataSourceId(FunctionSignature.getDataverseName(fid), fid.getName(), parameters);
}