You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2016/03/22 18:34:37 UTC
incubator-asterixdb git commit: Fix Asterix Installer Test Cases
Repository: incubator-asterixdb
Updated Branches:
refs/heads/master 36fbc4160 -> 30904b839
Fix Asterix Installer Test Cases
Many tests started to fail in installer after being
disabled for a long time. This change fixes the following:
1. asterix-installer: external library tests.
2. asterix-installer: recovery: query-after-restart.
Change-Id: Iefad5ac3ff754693025312f1e2fa01b08832514c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/737
Reviewed-by: Ian Maxon <im...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/30904b83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/30904b83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/30904b83
Branch: refs/heads/master
Commit: 30904b839e0da98dd7342e451618b4a67a4d9881
Parents: 36fbc41
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Tue Mar 22 19:25:10 2016 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Tue Mar 22 10:29:10 2016 -0700
----------------------------------------------------------------------
.gitignore | 1 +
asterix-app/pom.xml | 84 ++++++++++----------
.../feed-push-socket.4.server.aql | 2 +-
.../feed-with-external-function.1.ddl.aql | 58 ++++++++++++++
.../feed-with-external-function.2.lib.aql | 20 +++++
.../feed-with-external-function.3.update.aql | 32 ++++++++
.../feed-with-external-function.4.sleep.aql | 19 +++++
.../feed-with-external-function.5.query.aql | 32 ++++++++
.../feed-with-external-function.6.ddl.aql | 19 +++++
.../feed-with-external-function.7.lib.aql | 19 +++++
.../feed-with-meta-pk-in-meta.1.ddl.aql | 2 +-
..._dataset_with_meta_primary_index-1.1.ddl.aql | 2 +-
.../feed-with-external-function.1.adm | 12 +++
.../src/test/resources/runtimets/testsuite.xml | 5 ++
.../dataflow/RecordDataFlowController.java | 2 +
.../external/feed/watch/MonitoredBuffer.java | 12 +--
.../record/reader/kv/KVTestReaderFactory.java | 4 +-
.../operators/FeedMetaComputeNodePushable.java | 11 ++-
.../operators/FeedMetaOperatorDescriptor.java | 4 +-
.../apache/asterix/external/util/FeedUtils.java | 13 ++-
.../adapter/TestTypedAdapterFactory.java | 18 ++++-
asterix-installer/pom.xml | 51 +++++++++++-
.../installer/test/AbstractExecutionIT.java | 6 +-
.../test/AsterixExternalLibraryIT.java | 8 +-
.../installer/test/ManagixExecutionIT.java | 3 -
.../feed_ingest/feed_ingest.1.ddl.aql | 3 +-
.../feed_ingest/feed_ingest.3.query.aql | 32 --------
.../feed_ingest/feed_ingest.3.sleep.aql | 26 ++++++
.../feed_ingest/feed_ingest.4.query.aql | 32 ++++++++
.../record-parser/record-parser.1.ddl.aql | 8 +-
.../record-parser/record-parser.2.query.aql | 1 +
.../dataverseDataset/dataverseDataset.1.adm | 4 +-
.../external_index/external_index.1.adm | 6 +-
.../metadata/feeds/FeedMetadataUtil.java | 9 ++-
.../ExternalBTreeLocalResourceMetadata.java | 3 +-
35 files changed, 443 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index d802def..1b129d6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,7 @@ asterix-app/opt_parserts/
asterix-app/runtime_parserts/
asterix-installer/ittest/
asterix-installer/repliationtest/
+asterix-installer/src/test/resources/clusterts/asterix-installer-0.8.9-SNAPSHOT-binary-assembly/
build
asterix_logs
bin/
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/pom.xml
----------------------------------------------------------------------
diff --git a/asterix-app/pom.xml b/asterix-app/pom.xml
index 00e924f..d8e7aec 100644
--- a/asterix-app/pom.xml
+++ b/asterix-app/pom.xml
@@ -64,48 +64,48 @@
</executions>
</plugin>
<plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-external-library</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>src/test/resources/externallib</outputDirectory>
- <overwrite>true</overwrite>
- <resources>
- <resource>
- <directory>../asterix-external-data/target</directory>
- <includes>
- <include>testlib-zip-binary-assembly.zip</include>
- </includes>
- </resource>
- </resources>
- </configuration>
- </execution>
- <execution>
- <id>copy-beer-csv</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>data/csv</outputDirectory>
- <overwrite>true</overwrite>
- <resources>
- <resource>
- <directory>../asterix-external-data/src/test/resources</directory>
- <includes>
- <include>beer.csv</include>
- </includes>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
-</plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-external-library</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>src/test/resources/externallib</outputDirectory>
+ <overwrite>true</overwrite>
+ <resources>
+ <resource>
+ <directory>../asterix-external-data/target</directory>
+ <includes>
+ <include>testlib-zip-binary-assembly.zip</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-beer-csv</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/data/csv</outputDirectory>
+ <overwrite>true</overwrite>
+ <resources>
+ <resource>
+ <directory>../asterix-external-data/src/test/resources</directory>
+ <includes>
+ <include>beer.csv</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-push-socket/feed-push-socket.4.server.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-push-socket/feed-push-socket.4.server.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-push-socket/feed-push-socket.4.server.aql
index 578d458..5b610de 100644
--- a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-push-socket/feed-push-socket.4.server.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-push-socket/feed-push-socket.4.server.aql
@@ -23,4 +23,4 @@
* Expected Res : Success
* Date : 24th Feb 2016
*/
-start client 10001 file-client localhost data/twitter/tw_messages.adm 500 50 1000
\ No newline at end of file
+start client 10001 file-client localhost ../asterix-app/data/twitter/tw_messages.adm 500 50 1000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql
new file mode 100644
index 0000000..ce1d7fc
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a feed dataset that uses the feed simulator adapter.
+ The feed simulator simulates feed from a file in the local fs.
+ Associate with the feed an external user-defined function. The UDF
+ finds topics in each tweet. A topic is identified by a #.
+ Begin ingestion and apply external user defined function
+ * Expected Res : Success
+ * Date : 23rd Apr 2013
+ */
+
+drop dataverse externallibtest if exists;
+create dataverse externallibtest;
+use dataverse externallibtest;
+
+create type TweetInputType as closed {
+ id: string,
+ username : string,
+ location : string,
+ text : string,
+ timestamp : string
+}
+
+create type TweetOutputType as closed {
+ id: string,
+ username : string,
+ location : string,
+ text : string,
+ timestamp : string,
+ topics : {{string}}
+}
+
+create feed TweetFeed
+using localfs
+(("type-name"="TweetInputType"),
+("path"="asterix_nc1://data/twitter/obamatweets.adm"),
+("format"="adm"))
+apply function testlib#parseTweet;
+
+create dataset TweetsFeedIngest(TweetOutputType)
+primary key id;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql
new file mode 100644
index 0000000..124038a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+install externallibtest testlib src/test/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql
new file mode 100644
index 0000000..ea1a379
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a feed dataset that uses the feed simulator adapter.
+ The feed simulator simulates feed from a file in the local fs.
+ Associate with the feed an external user-defined function. The UDF
+ finds topics in each tweet. A topic is identified by a #.
+ Begin ingestion and apply external user defined function
+ * Expected Res : Success
+ * Date : 23rd Apr 2013
+ */
+use dataverse externallibtest;
+
+set wait-for-completion-feed "true";
+
+connect feed TweetFeed to dataset TweetsFeedIngest;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql
new file mode 100644
index 0000000..af2f691
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+10000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql
new file mode 100644
index 0000000..8879fa8
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a feed dataset that uses the feed simulator adapter.
+ The feed simulator simulates feed from a file in the local fs.
+ Associate with the feed an external user-defined function. The UDF
+ finds topics in each tweet. A topic is identified by a #.
+ Begin ingestion and apply external user defined function
+ * Expected Res : Success
+ * Date : 23rd Apr 2013
+ */
+use dataverse externallibtest;
+
+for $x in dataset TweetsFeedIngest
+order by $x.id
+return $x
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql
new file mode 100644
index 0000000..2a7acef
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse externallibtest if exists;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql
new file mode 100644
index 0000000..86af80f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+uninstall externallibtest testlib
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
index ec4a712..de9446a 100644
--- a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
@@ -46,7 +46,7 @@ create feed KVStream using adapter(
("parser"="record-with-metadata"),
("type-name"="DocumentType"),
("meta-type-name"="KVMetaType"),
- ("path"="asterix_nc1://data/csv/beer.csv"),
+ ("path"="asterix_nc1://target/data/csv/beer.csv"),
("format"="csv"),
("delimiter"=","),
("record-format"="adm"),
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
index 718d2a6..26d4b09 100644
--- a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
@@ -41,7 +41,7 @@ create feed KVStream using adapter(
("parser"="record-with-metadata"),
("type-name"="DocumentType"),
("meta-type-name"="KVMetaType"),
- ("path"="asterix_nc1://data/csv/beer.csv"),
+ ("path"="asterix_nc1://target/data/csv/beer.csv"),
("format"="csv"),
("delimiter"=","),
("record-format"="adm"),
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
new file mode 100644
index 0000000..1291213
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
@@ -0,0 +1,12 @@
+{ "id": "nc1:1", "username": "BronsonMike", "location": "", "text": "@GottaLaff @reutersus Christie and obama just foul weather friends", "timestamp": "Thu Dec 06 16:53:06 PST 2012", "topics": {{ }} }
+{ "id": "nc1:100", "username": "KidrauhlProuds", "location": "", "text": "RT @01Direclieber: A filha do Michael Jackson uma Belieber,a filha do Eminem e uma Belieber,as filhas de Obama sao Beliebers, e a filha do meu pai e Belieber", "timestamp": "Thu Dec 06 16:53:16 PST 2012", "topics": {{ }} }
+{ "id": "nc1:102", "username": "jaysauce82", "location": "", "text": "Not voting for President Obama #BadDecision", "timestamp": "Thu Dec 06 16:53:16 PST 2012", "topics": {{ "#BadDecision" }} }
+{ "id": "nc1:104", "username": "princeofsupras", "location": "", "text": "RT @01Direclieber: A filha do Michael Jackson e uma Belieber,a filha do Eminem e uma Belieber,as filhas de Obama sao Beliebers, e a filha do meu pai e Belieber", "timestamp": "Thu Dec 06 16:53:15 PST 2012", "topics": {{ }} }
+{ "id": "nc1:106", "username": "GulfDogs", "location": "", "text": "Obama Admin Knew Libyan Terrorists Had US-Provided Weaponsteaparty #tcot #ccot #NewGuards #BreitbartArmy #patriotwttp://t.co/vJxzrQUE", "timestamp": "Thu Dec 06 16:53:14 PST 2012", "topics": {{ "#tcot", "#ccot", "#NewGuards", "#BreitbartArmy", "#patriotwttp://t.co/vJxzrQUE" }} }
+{ "id": "nc1:108", "username": "Laugzpz", "location": "", "text": "@AlfredoJalife Maestro Obama se hace de la vista gorda, es un acuerdo de siempre creo yo.", "timestamp": "Thu Dec 06 16:53:14 PST 2012", "topics": {{ }} }
+{ "id": "nc1:11", "username": "magarika", "location": "", "text": "RT @ken24xavier: Obama tells SOROS - our plan is ALMOST finished http://t.co/WvzK0GtU", "timestamp": "Thu Dec 06 16:53:05 PST 2012", "topics": {{ }} }
+{ "id": "nc1:111", "username": "ToucanMall", "location": "", "text": "RT @WorldWar3Watch: Michelle Obama Gets More Grammy Nominations Than Justin ... #Obama #WW3 http://t.co/0Wv2GKij", "timestamp": "Thu Dec 06 16:53:13 PST 2012", "topics": {{ "#Obama", "#WW3" }} }
+{ "id": "nc1:113", "username": "ToucanMall", "location": "", "text": "RT @ObamaPalooza: Tiffany Shared What $2,000 Meant to Her ... and the President Stopped by to Talk About It http://t.co/sgT7lsNV #Obama", "timestamp": "Thu Dec 06 16:53:12 PST 2012", "topics": {{ "#Obama" }} }
+{ "id": "nc1:115", "username": "thewildpitch", "location": "", "text": "RT @RevkahJC: Dennis Miller: Obama Should Just Say He Wants To Tax Successful People http://t.co/Ihlemy9Y", "timestamp": "Thu Dec 06 16:53:11 PST 2012", "topics": {{ }} }
+{ "id": "nc1:117", "username": "Rnugent24", "location": "", "text": "RT @ConservativeQuo: unemployment is above 8% again. I wonder how long it will take for Obama to start blaming Bush? 3-2-1 #tcot #antiobama", "timestamp": "Thu Dec 06 16:53:10 PST 2012", "topics": {{ "#tcot", "#antiobama" }} }
+{ "id": "nc1:119", "username": "ToucanMall", "location": "", "text": "RT @Newitrsdotcom: I hope #Obama will win re-election... Other four years without meaningless #wars", "timestamp": "Thu Dec 06 16:53:09 PST 2012", "topics": {{ "#Obama", "#wars" }} }
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/testsuite.xml
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 115cf92..fbc821a 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -30,6 +30,11 @@
QueryFileExtension=".aql">
<test-group name="feeds">
<test-case FilePath="feeds">
+ <compilation-unit name="feed-with-external-function">
+ <output-dir compare="Text">feed-with-external-function</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
<compilation-unit name="change-feed-with-meta-pk-in-meta">
<output-dir compare="Text">change-feed-with-meta-pk-in-meta</output-dir>
</compilation-unit>
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
index 99654d0..915c343 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
@@ -28,9 +28,11 @@ import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.log4j.Logger;
public class RecordDataFlowController<T> extends AbstractDataFlowController {
+ private static final Logger LOGGER = Logger.getLogger(RecordDataFlowController.class.getName());
protected final IRecordDataParser<T> dataParser;
protected final IRecordReader<? extends T> recordReader;
protected final int numOfTupleFields;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
index b36fb6a..67e6295 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
@@ -26,13 +26,13 @@ import java.util.logging.Level;
import org.apache.asterix.external.feed.api.IExceptionHandler;
import org.apache.asterix.external.feed.api.IFeedMetricCollector;
-import org.apache.asterix.external.feed.api.IFrameEventCallback;
-import org.apache.asterix.external.feed.api.IFramePostProcessor;
-import org.apache.asterix.external.feed.api.IFramePreprocessor;
import org.apache.asterix.external.feed.api.IFeedMetricCollector.MetricType;
import org.apache.asterix.external.feed.api.IFeedMetricCollector.ValueType;
import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
import org.apache.asterix.external.feed.api.IFrameEventCallback.FrameEvent;
+import org.apache.asterix.external.feed.api.IFramePostProcessor;
+import org.apache.asterix.external.feed.api.IFramePreprocessor;
import org.apache.asterix.external.feed.dataflow.DataBucket;
import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
import org.apache.asterix.external.feed.dataflow.StorageFrameHandler;
@@ -170,7 +170,7 @@ public abstract class MonitoredBuffer extends MessageReceiver<DataBucket> {
this.timer.scheduleAtFixedRate(monitorInputQueueLengthTask, 0, INPUT_QUEUE_MEASURE_FREQUENCY);
}
- if (logInflowOutflowRate || reportInflowRate || reportOutflowRate) {
+ if (reportInflowRate || reportOutflowRate) {
this.logInflowOutflowRateTask = new LogInputOutputRateTask(this, logInflowOutflowRate, reportInflowRate,
reportOutflowRate);
this.timer.scheduleAtFixedRate(logInflowOutflowRateTask, 0, LOG_INPUT_OUTPUT_RATE_FREQUENCY);
@@ -188,7 +188,7 @@ public abstract class MonitoredBuffer extends MessageReceiver<DataBucket> {
if (processingRateTask != null) {
processingRateTask.cancel();
}
- if (logInflowOutflowRate || reportInflowRate || reportOutflowRate) {
+ if (reportInflowRate || reportOutflowRate) {
metricCollector.removeReportSender(inflowReportSenderId);
metricCollector.removeReportSender(outflowReportSenderId);
logInflowOutflowRateTask.cancel();
@@ -244,7 +244,7 @@ public abstract class MonitoredBuffer extends MessageReceiver<DataBucket> {
}
public void sendReport(ByteBuffer frame) {
- if ((logInflowOutflowRate || reportInflowRate) && !(inputHandler.getMode().equals(Mode.PROCESS_BACKLOG)
+ if ((reportInflowRate) && !(inputHandler.getMode().equals(Mode.PROCESS_BACKLOG)
|| inputHandler.getMode().equals(Mode.PROCESS_SPILL))) {
inflowFta.reset(frame);
metricCollector.sendReport(inflowReportSenderId, inflowFta.getTupleCount());
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
index adaa525..f5afef6 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
@@ -69,8 +69,8 @@ public class KVTestReaderFactory implements IRecordReaderFactory<DCPRequest> {
@Override
public IRecordReader<? extends DCPRequest> createRecordReader(final IHyracksTaskContext ctx, final int partition) {
return new KVTestReader(partition, bucket, schedule,
- (int) Math.ceil((double) numOfRecords / (double) getPartitionConstraint().getLocations().length),
- deleteCycle, upsertCycle, (numOfRecords / numOfReaders) * partition);
+ (int) Math.ceil((double) numOfRecords / (double) numOfReaders), deleteCycle, upsertCycle,
+ (numOfRecords / numOfReaders) * partition);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
index 66b520f..210a94d 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
@@ -95,9 +95,14 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp
private ByteBuffer message = ByteBuffer.allocate(MessagingFrameTupleAppender.MAX_MESSAGE_SIZE);
+ private final FeedMetaOperatorDescriptor opDesc;
+
+ private final IRecordDescriptorProvider recordDescProvider;
+
public FeedMetaComputeNodePushable(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
int partition, int nPartitions, IOperatorDescriptor coreOperator, FeedConnectionId feedConnectionId,
- Map<String, String> feedPolicyProperties, String operationId) throws HyracksDataException {
+ Map<String, String> feedPolicyProperties, String operationId,
+ FeedMetaOperatorDescriptor feedMetaOperatorDescriptor) throws HyracksDataException {
this.ctx = ctx;
this.coreOperator = (AbstractUnaryInputUnaryOutputOperatorNodePushable) ((IActivity) coreOperator)
.createPushRuntime(ctx, recordDescProvider, partition, nPartitions);
@@ -108,6 +113,8 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp
this.feedManager = (IFeedManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
.getApplicationObject()).getFeedManager();
ctx.setSharedObject(message);
+ this.opDesc = feedMetaOperatorDescriptor;
+ this.recordDescProvider = recordDescProvider;
}
@Override
@@ -129,7 +136,7 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp
}
private void initializeNewFeedRuntime(FeedRuntimeId runtimeId) throws Exception {
- this.fta = new FrameTupleAccessor(recordDesc);
+ this.fta = new FrameTupleAccessor(recordDescProvider.getInputRecordDescriptor(opDesc.getActivityId(), 0));
this.inputSideHandler = new FeedRuntimeInputHandler(ctx, connectionId, runtimeId, coreOperator,
policyEnforcer.getFeedPolicyAccessor(), policyEnforcer.getFeedPolicyAccessor().bufferingEnabled(), fta,
recordDesc, feedManager, nPartitions);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
index 354636d..6591795 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
@@ -90,12 +90,12 @@ public class FeedMetaOperatorDescriptor extends AbstractSingleActivityOperatorDe
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
- throws HyracksDataException {
+ throws HyracksDataException {
IOperatorNodePushable nodePushable = null;
switch (runtimeType) {
case COMPUTE:
nodePushable = new FeedMetaComputeNodePushable(ctx, recordDescProvider, partition, nPartitions,
- coreOperator, feedConnectionId, feedPolicyProperties, operandId);
+ coreOperator, feedConnectionId, feedPolicyProperties, operandId, this);
break;
case STORE:
nodePushable = new FeedMetaStoreNodePushable(ctx, recordDescProvider, partition, nPartitions,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
index c9fc5fc..5ab41af 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
@@ -45,16 +45,17 @@ public class FeedUtils {
return dataverseName + File.separator + feedName;
}
- public static FileSplit[] splitsForAdapter(String dataverseName, String feedName, String nodeName, int partition) {
+ public static FileSplit splitsForAdapter(String dataverseName, String feedName, int partition,
+ ClusterPartition[] nodePartitions) {
File relPathFile = new File(prepareDataverseFeedName(dataverseName, feedName));
String storageDirName = AsterixClusterProperties.INSTANCE.getStorageDirectoryName();
- ClusterPartition nodePartition = AsterixClusterProperties.INSTANCE.getNodePartitions(nodeName)[0];
+ ClusterPartition nodePartition = nodePartitions[0];
String storagePartitionPath = StoragePathUtil.prepareStoragePartitionPath(storageDirName,
nodePartition.getPartitionId());
// format: 'storage dir name'/partition_#/dataverse/feed/adapter_#
File f = new File(storagePartitionPath + File.separator + relPathFile + File.separator
+ StoragePathUtil.ADAPTER_INSTANCE_PREFIX + partition);
- return new FileSplit[] { StoragePathUtil.getFileSplitForClusterPartition(nodePartition, f) };
+ return StoragePathUtil.getFileSplitForClusterPartition(nodePartition, f);
}
public static FileSplit[] splitsForAdapter(String dataverseName, String feedName,
@@ -93,6 +94,12 @@ public class FeedUtils {
feedLogFileSplits[partition].getIODeviceId(), ctx.getIOManager()).getFile());
}
+ public static FeedLogManager getFeedLogManager(IHyracksTaskContext ctx, FileSplit feedLogFileSplit)
+ throws HyracksDataException {
+ return new FeedLogManager(FeedUtils.getAbsoluteFileRef(feedLogFileSplit.getLocalFile().getFile().getPath(),
+ feedLogFileSplit.getIODeviceId(), ctx.getIOManager()).getFile());
+ }
+
public static void processFeedMessage(ByteBuffer input, ByteBuffer message, FrameTupleAccessor fta) {
// read the message and reduce the number of tuples
fta.reset(input);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
index 9082a72..7da6389 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
+import org.apache.asterix.common.cluster.ClusterPartition;
+import org.apache.asterix.common.config.IAsterixPropertiesProvider;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.api.IDataSourceAdapter;
@@ -36,6 +38,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.std.file.ITupleParser;
import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
@@ -74,12 +77,19 @@ public class TestTypedAdapterFactory implements IAdapterFactory {
ADMDataParser parser;
ITupleForwarder forwarder;
ArrayTupleBuilder tb;
+ IAsterixPropertiesProvider propertiesProvider = (IAsterixPropertiesProvider) ((NodeControllerService) ctx
+ .getJobletContext().getApplicationContext().getControllerService()).getApplicationContext()
+ .getApplicationObject();
+ ClusterPartition[] nodePartitions = propertiesProvider.getMetadataProperties().getNodePartitions()
+ .get(nodeId);
try {
parser = new ADMDataParser(outputType, true);
- forwarder = DataflowUtils.getTupleForwarder(configuration,
- FeedUtils.getFeedLogManager(ctx, partition,
- FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration),
- ExternalDataUtils.getFeedName(configuration), nodeId, partition)));
+ forwarder = DataflowUtils
+ .getTupleForwarder(configuration,
+ FeedUtils.getFeedLogManager(ctx,
+ FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration),
+ ExternalDataUtils.getFeedName(configuration), partition,
+ nodePartitions)));
tb = new ArrayTupleBuilder(1);
} catch (Exception e) {
throw new HyracksDataException(e);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/pom.xml
----------------------------------------------------------------------
diff --git a/asterix-installer/pom.xml b/asterix-installer/pom.xml
index 96e8fa1..cdda6b7 100644
--- a/asterix-installer/pom.xml
+++ b/asterix-installer/pom.xml
@@ -138,8 +138,57 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-external-library</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/resources/externallib</outputDirectory>
+ <overwrite>true</overwrite>
+ <resources>
+ <resource>
+ <directory>../asterix-external-data/target</directory>
+ <includes>
+ <include>testlib-zip-binary-assembly.zip</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-test-data</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/data</outputDirectory>
+ <overwrite>true</overwrite>
+ <resources>
+ <resource>
+ <directory>../asterix-external-data/src/test/resources</directory>
+ <includes>
+ <include>beer.csv</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>../asterix-app/data/external-parser</directory>
+ <includes>
+ <include>jobads.new</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
- </build>
+ </build>
<profiles>
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java b/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
index a6d544e..ba2d7f1 100644
--- a/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
+++ b/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
@@ -72,7 +72,7 @@ public abstract class AbstractExecutionIT {
.copyFile(
new File(StringUtils.join(new String[] { "src", "test", "resources", "integrationts",
"asterix-configuration.xml" }, File.separator)),
- new File(AsterixInstallerIntegrationUtil.getManagixHome() + "/conf/asterix-configuration.xml"));
+ new File(AsterixInstallerIntegrationUtil.getManagixHome() + "/conf/asterix-configuration.xml"));
AsterixLifecycleIT.setUp();
@@ -80,6 +80,10 @@ public abstract class AbstractExecutionIT {
new File(StringUtils.join(new String[] { "..", "asterix-app", "data" }, File.separator)),
new File(AsterixInstallerIntegrationUtil.getManagixHome() + "/clusters/local/working_dir/data"));
+ FileUtils.copyDirectoryStructure(new File(StringUtils.join(new String[] { "target", "data" }, File.separator)),
+ new File(AsterixInstallerIntegrationUtil.getManagixHome()
+ + "/clusters/local/working_dir/target/data/csv"));
+
// Set the node resolver to be the identity resolver that expects node names
// to be node controller ids; a valid assumption in test environment.
System.setProperty(ExternalDataConstants.NODE_RESOLVER_FACTORY_PROPERTY,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java b/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
index 1da01c3..506e31f 100644
--- a/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
+++ b/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
@@ -35,8 +35,8 @@ public class AsterixExternalLibraryIT {
private static final String LIBRARY_DATAVERSE = "externallibtest";
private static final String PATH_BASE = "src/test/resources/integrationts/library";
private static final String PATH_ACTUAL = "ittest/";
- private static final String LIBRARY_PATH = "asterix-external-data" + File.separator + "target" + File.separator
- + "testlib-zip-binary-assembly.zip";
+ private static final String LIBRARY_PATH = "target" + File.separator + "resources" + File.separator + "externallib"
+ + File.separator + "testlib-zip-binary-assembly.zip";
private static final Logger LOGGER = Logger.getLogger(AsterixExternalLibraryIT.class.getName());
private static List<TestCaseContext> testCaseCollection;
private final TestExecutor testExecutor = new TestExecutor();
@@ -46,8 +46,8 @@ public class AsterixExternalLibraryIT {
try {
AsterixInstallerIntegrationUtil.init();
File asterixInstallerProjectDir = new File(System.getProperty("user.dir"));
- String asterixExternalLibraryPath = asterixInstallerProjectDir.getParentFile().getAbsolutePath()
- + File.separator + LIBRARY_PATH;
+ String asterixExternalLibraryPath = asterixInstallerProjectDir.getAbsolutePath() + File.separator
+ + LIBRARY_PATH;
LOGGER.info("Installing library :" + LIBRARY_NAME + " located at " + asterixExternalLibraryPath
+ " in dataverse " + LIBRARY_DATAVERSE);
AsterixInstallerIntegrationUtil.installLibrary(LIBRARY_NAME, LIBRARY_DATAVERSE, asterixExternalLibraryPath);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/java/org/apache/asterix/installer/test/ManagixExecutionIT.java
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/java/org/apache/asterix/installer/test/ManagixExecutionIT.java b/asterix-installer/src/test/java/org/apache/asterix/installer/test/ManagixExecutionIT.java
index 14a1a58..6df34ce 100644
--- a/asterix-installer/src/test/java/org/apache/asterix/installer/test/ManagixExecutionIT.java
+++ b/asterix-installer/src/test/java/org/apache/asterix/installer/test/ManagixExecutionIT.java
@@ -24,10 +24,7 @@ import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
public class ManagixExecutionIT extends AbstractExecutionIT {
- private TestCaseContext tcCtx;
-
public ManagixExecutionIT(TestCaseContext tcCtx) {
super(tcCtx);
- this.tcCtx = tcCtx;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
index f9a6eda..dbe3cfa 100644
--- a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
+++ b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
@@ -48,8 +48,7 @@ create feed TweetFeed
using localfs
(("type-name"="TweetInputType"),
("path"="asterix_nc1://../../../../../../asterix-app/data/twitter/obamatweets.adm"),
-("format"="adm"),
-("tuple-interval"="10"))
+("format"="adm"))
apply function testlib#parseTweet;
create dataset TweetsFeedIngest(TweetOutputType)
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql
deleted file mode 100644
index e4e1b45..0000000
--- a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Create a feed dataset that uses the feed simulator adapter.
- The feed simulator simulates feed from a file in the local fs.
- Associate with the feed an external user-defined function. The UDF
- finds topics in each tweet. A topic is identified by a #.
- Begin ingestion and apply external user defined function
- * Expected Res : Success
- * Date : 23rd Apr 2013
- */
-use dataverse externallibtest;
-
-for $x in dataset TweetsFeedIngest
-order by $x.id
-return $x
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql
new file mode 100644
index 0000000..18bbbbc
--- /dev/null
+++ b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a socket feed with a client that pushes
+ * 10 records. The feed is connected to a dataset that is then
+ * queried for the data.
+ * Expected Res : Success
+ * Date : 24th Feb 2016
+ */
+10000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql
new file mode 100644
index 0000000..e4e1b45
--- /dev/null
+++ b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a feed dataset that uses the feed simulator adapter.
+ The feed simulator simulates feed from a file in the local fs.
+ Associate with the feed an external user-defined function. The UDF
+ finds topics in each tweet. A topic is identified by a #.
+ Begin ingestion and apply external user defined function
+ * Expected Res : Success
+ * Date : 23rd Apr 2013
+ */
+use dataverse externallibtest;
+
+for $x in dataset TweetsFeedIngest
+order by $x.id
+return $x
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
index d523247..09db37c 100644
--- a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
+++ b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
@@ -24,10 +24,12 @@
use dataverse externallibtest;
create type Classad as open {
- GlobalJobId: int64
+ GlobalJobId: string
}
create external dataset Condor(Classad) using localfs(
-("path"="asterix_nc1://data/external-parser/jobads.new"),
-("format"="adm"),
+("path"="asterix_nc1://../../../../data/jobads.new"),
+("format"="semi-structured"),
+("record-start"="["),
+("record-end"="]"),
("parser"="testlib#org.apache.asterix.external.library.ClassAdParserFactory"));
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
index ca2d832..26f980f 100644
--- a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
+++ b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
@@ -24,4 +24,5 @@
use dataverse externallibtest;
for $x in dataset Condor
+order by $x.GlobalJobId
return $x;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm b/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
index d4e5619..2c4af14 100644
--- a/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
+++ b/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
@@ -1,2 +1,2 @@
-{ "DataverseName": "Metadata", "DataFormat": "org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr 25 11:17:56 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "externallibtest", "DataFormat": "org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr 25 11:18:12 PDT 2013", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DataFormat": "org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr 25 11:17:56 PDT 2013", "PendingOp": 0i32 }
+{ "DataverseName": "externallibtest", "DataFormat": "org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr 25 11:18:12 PDT 2013", "PendingOp": 0i32 }
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm b/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
index 36b836c..3ee4a1e 100644
--- a/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
+++ b/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
@@ -1,4 +1,2 @@
-[ { "id": 3, "name": "Samuel", "age": 22 }
-, { "id": 10, "name": "David", "age": 22 }
- ]
-
+{ "id": 3, "name": "Samuel", "age": 22 }
+{ "id": 10, "name": "David", "age": 22 }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index 78c6587..8561bf5 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -214,12 +214,15 @@ public class FeedMetadataUtil {
Map<ConnectorDescriptorId, ConnectorDescriptorId> connectorMapping = new HashMap<ConnectorDescriptorId, ConnectorDescriptorId>();
for (Entry<ConnectorDescriptorId, IConnectorDescriptor> entry : spec.getConnectorMap().entrySet()) {
IConnectorDescriptor connDesc = entry.getValue();
+ ConnectorDescriptorId newConnId;
if (connDesc instanceof MToNPartitioningConnectorDescriptor) {
MToNPartitioningConnectorDescriptor m2nConn = (MToNPartitioningConnectorDescriptor) connDesc;
connDesc = new MToNPartitioningWithMessageConnectorDescriptor(altered,
m2nConn.getTuplePartitionComputerFactory());
+ newConnId = connDesc.getConnectorId();
+ } else {
+ newConnId = altered.createConnectorDescriptor(connDesc);
}
- ConnectorDescriptorId newConnId = altered.createConnectorDescriptor(connDesc);
connectorMapping.put(entry.getKey(), newConnId);
}
@@ -466,7 +469,7 @@ public class FeedMetadataUtil {
@SuppressWarnings("rawtypes")
public static Triple<IAdapterFactory, RecordDescriptor, IDataSourceAdapter.AdapterType> getPrimaryFeedFactoryAndOutput(
Feed feed, FeedPolicyAccessor policyAccessor, MetadataTransactionContext mdTxnCtx)
- throws AlgebricksException {
+ throws AlgebricksException {
// This method needs to be re-visited
String adapterName = null;
DatasourceAdapter adapterEntity = null;
@@ -606,7 +609,7 @@ public class FeedMetadataUtil {
public static String getSecondaryFeedOutput(Feed feed, FeedPolicyAccessor policyAccessor,
MetadataTransactionContext mdTxnCtx)
- throws AlgebricksException, MetadataException, RemoteException, ACIDException {
+ throws AlgebricksException, MetadataException, RemoteException, ACIDException {
String outputType = null;
String primaryFeedName = feed.getSourceFeedName();
Feed primaryFeed = MetadataManager.INSTANCE.getFeed(mdTxnCtx, feed.getDataverseName(), primaryFeedName);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
----------------------------------------------------------------------
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
index 33b7f30..b76e829 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
@@ -45,8 +45,9 @@ public class ExternalBTreeLocalResourceMetadata extends LSMBTreeLocalResourceMet
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) {
+ int partition, int ioDeviceNum) {
FileReference file = new FileReference(new File(filePath));
+
LSMBTree lsmBTree = LSMBTreeUtils.createExternalBTree(file, runtimeContextProvider.getBufferCache(),
runtimeContextProvider.getFileMapManager(), typeTraits, cmpFactories, bloomFilterKeyFields,
runtimeContextProvider.getBloomFilterFalsePositiveRate(),