You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by xi...@apache.org on 2018/11/16 06:08:34 UTC
asterixdb git commit: [ASTERIXDB-2480][ING] Fix http-feed
compatibility with JSON parser
Repository: asterixdb
Updated Branches:
refs/heads/master 0dec33a96 -> 19e9d8290
[ASTERIXDB-2480][ING] Fix http-feed compatibility with JSON parser
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
The generic record used in current http-feed returns length of "-1"
which will cause the JSON parser to fail. For compatibility
consideration, we should use CharArrayRecord instead.
Change-Id: I719c98376583a3316dbc291696df65a97dd4b26e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3026
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wa...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/19e9d829
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/19e9d829
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/19e9d829
Branch: refs/heads/master
Commit: 19e9d8290d64d4a29c1de78df3eaf8ee32553eed
Parents: 0dec33a
Author: Xikui Wang <xk...@gmail.com>
Authored: Thu Nov 15 13:55:02 2018 -0800
Committer: Xikui Wang <xk...@gmail.com>
Committed: Thu Nov 15 22:08:19 2018 -0800
----------------------------------------------------------------------
.../http_feed_json/http_feed_json.1.ddl.sqlpp | 38 ++++++++++++++++++++
.../http_feed_json.2.update.sqlpp | 24 +++++++++++++
.../http_feed_json/http_feed_json.3.post.http | 20 +++++++++++
.../http_feed_json.4.update.sqlpp | 21 +++++++++++
.../http_feed_json/http_feed_json.5.query.sqlpp | 22 ++++++++++++
.../http_feed_json/http_feed_json.6.ddl.sqlpp | 21 +++++++++++
.../feeds/http_feed_json/http_feed.1.adm | 1 +
.../feeds/http_feed_json/http_feed.2.adm | 1 +
.../resources/runtimets/testsuite_sqlpp.xml | 5 +++
.../reader/http/HttpServerRecordReader.java | 6 ++--
10 files changed, 156 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp
new file mode 100644
index 0000000..94a773c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type WorrisomeNotificationType as {
+ dataverseName: string,
+ channelName: string
+};
+create type WorrisomeNotificationStoreType as {
+ id: uuid
+};
+create feed WorrisomeTweetsFeed with {
+ "adapter-name" : "http_adapter",
+ "addresses" : "asterix_nc1:10001",
+ "address-type" : "NC",
+ "type-name" : "WorrisomeNotificationType",
+ "format" : "json"
+};
+
+create dataset WorrisomeNotifications(WorrisomeNotificationStoreType) primary key id autogenerated;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp
new file mode 100644
index 0000000..cc180a3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+set `wait-for-completion-feed` "false";
+
+connect feed WorrisomeTweetsFeed to dataset WorrisomeNotifications;
+start feed WorrisomeTweetsFeed;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http
new file mode 100644
index 0000000..2169163
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+nc:asterix_nc1:10001 /
+--body={ "dataverseName":"dhs", "channelName":"WorrisomeTweetsNearMonumentsChannel", "channelExecutionTime":"2018-11-12T18:43:48.508Z"}{ "dataverseName":"dhs", "channelName":"WorrisomeTweetsNearMonumentsChannel", "channelExecutionTime":"2018-11-12T18:43:48.508Z"}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp
new file mode 100644
index 0000000..90c3f8e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+stop feed WorrisomeTweetsFeed;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp
new file mode 100644
index 0000000..65d6faf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select value count(t) from WorrisomeNotifications as t;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp
new file mode 100644
index 0000000..c17ef97
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+drop dataverse test;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm
@@ -0,0 +1 @@
+2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm
@@ -0,0 +1 @@
+2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index e52449c..72a96a6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -9984,6 +9984,11 @@
<output-dir compare="Text">http_feed</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="http_feed_json">
+ <output-dir compare="Text">http_feed_json</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="meta">
<test-case FilePath="meta">
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java
index 2979641..b466614 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java
@@ -27,7 +27,7 @@ import java.util.concurrent.LinkedBlockingQueue;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.input.record.GenericRecord;
+import org.apache.asterix.external.input.record.CharArrayRecord;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
@@ -49,7 +49,7 @@ public class HttpServerRecordReader implements IRecordReader<char[]> {
private static final String DEFAULT_ENTRY_POINT = "/";
private static final int DEFAULT_QUEUE_SIZE = 128;
private LinkedBlockingQueue<String> inputQ;
- private GenericRecord<char[]> record;
+ private CharArrayRecord record;
private boolean closed = false;
private WebManager webManager;
private HttpServer webServer;
@@ -57,7 +57,7 @@ public class HttpServerRecordReader implements IRecordReader<char[]> {
public HttpServerRecordReader(int port, String entryPoint, int queueSize, HttpServerConfig httpServerConfig)
throws Exception {
this.inputQ = new LinkedBlockingQueue<>(queueSize > 0 ? queueSize : DEFAULT_QUEUE_SIZE);
- this.record = new GenericRecord<>();
+ this.record = new CharArrayRecord(0);
webManager = new WebManager();
webServer = new HttpServer(webManager.getBosses(), webManager.getWorkers(), port, httpServerConfig);
webServer.addServlet(new HttpFeedServlet(webServer.ctx(),