You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by sj...@apache.org on 2016/09/07 23:22:43 UTC
[1/4] vxquery git commit: VXQUERY-207 VXQUERY-209 Parallel Index
creation and access, and index centralization
Repository: vxquery
Updated Branches:
refs/heads/master 6bee358bc -> eb76640fc
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex2.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex2.xq
deleted file mode 100644
index bf19ee9..0000000
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex2.xq
+++ /dev/null
@@ -1,24 +0,0 @@
-(: Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License. :)
-
-(: Search Lucene Index :)
-(: Find all reading for hurricane force wind warning or extreme wind warning. :)
-(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :)
-(: meters per second). (Wind value is in tenth of a meter per second) :)
-for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
-where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
-return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex3.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex3.xq
deleted file mode 100644
index 28cf019..0000000
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex3.xq
+++ /dev/null
@@ -1,27 +0,0 @@
-(: Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License. :)
-
-(: Search Lucene Index :)
-(: Find the annual precipitation (PRCP) for a Seattle using the airport :)
-(: station (US000000002) for 2002. :)
-fn:sum(
- for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
- where $r/station eq "GHCND:US000000002"
- and $r/dataType eq "PRCP"
- and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
- return $r/value
-) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex4.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex4.xq
deleted file mode 100644
index 2b75cf4..0000000
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex4.xq
+++ /dev/null
@@ -1,24 +0,0 @@
-(: Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License. :)
-
-(: Search Lucene Index :)
-(: Find the highest recorded temperature (TMAX) in Celsius. :)
-fn:max(
- for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
- where $r/dataType eq "TMAX"
- return $r/value
-) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex5.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex5.xq
deleted file mode 100644
index e83484a..0000000
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex5.xq
+++ /dev/null
@@ -1,23 +0,0 @@
-(: Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License. :)
-
-(: Search Lucene Index :)
-(: Find all the weather readings for Washington state for a specific day :)
-(: 2002-2-2. :)
-for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
-where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
-return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex6.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex6.xq
deleted file mode 100644
index 04f6672..0000000
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex6.xq
+++ /dev/null
@@ -1,23 +0,0 @@
-(: Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License. :)
-
-(: Search Lucene Index :)
-(: Find all the weather readings for Washington state for a specific day :)
-(: 2002-2-2. :)
-for $s in collection-from-index("target/tmp/indexFolder", "/stationCollection/station")/station
-where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
-return $s
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex7.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex7.xq
deleted file mode 100644
index e471baa..0000000
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex7.xq
+++ /dev/null
@@ -1,27 +0,0 @@
-(: Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License. :)
-
-(: Search Lucene Index :)
-(: Find all the weather readings for Washington state for a specific day :)
-(: 2002-2-2. :)
-for $s in collection-from-index("target/tmp/indexFolder", "/stationCollection/station")/station
-for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
-
-where $s/id eq $r/station
- and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
- and xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
-return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml b/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
index 763d8c3..fc668ba 100644
--- a/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
+++ b/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
@@ -20,54 +20,164 @@
<title>Indexing</title>
<description/>
</GroupInfo>
- <test-case name="create-index" FilePath="Indexing/" Creator="Steven Jacobs">
+ <test-case name="create-index" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
<description>Create Lucene Index from Collection.</description>
<query name="createIndex" date="2016-05-26"/>
<output-file compare="Text">createIndex.txt</output-file>
</test-case>
- <test-case name="use-index-1" FilePath="Indexing/" Creator="Steven Jacobs">
+ <test-case name="show-index-1" FilePath="Indexing/Partition-1/" Creator="Menaka Jayawardena">
+ <description>Show indexes after creating index.</description>
+ <query name="showIndex" date="2016-08-18"/>
+ <output-file compare="Text">showIndex1.txt</output-file>
+ </test-case>
+ <test-case name="use-index-1" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex1" date="2016-05-26"/>
<output-file compare="Text">useIndex1.txt</output-file>
</test-case>
- <test-case name="use-index-2" FilePath="Indexing/" Creator="Steven Jacobs">
+ <test-case name="use-index-2" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex2" date="2016-05-26"/>
<output-file compare="Text">useIndex2.txt</output-file>
</test-case>
- <test-case name="use-index-3" FilePath="Indexing/" Creator="Steven Jacobs">
+ <test-case name="use-index-3" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex3" date="2016-05-26"/>
<output-file compare="Text">useIndex3.txt</output-file>
</test-case>
- <test-case name="use-index-4" FilePath="Indexing/" Creator="Steven Jacobs">
+ <test-case name="use-index-4" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex4" date="2016-05-26"/>
<output-file compare="Text">useIndex4.txt</output-file>
</test-case>
- <test-case name="use-index-5" FilePath="Indexing/" Creator="Steven Jacobs">
+ <test-case name="use-index-5" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex5" date="2016-05-26"/>
<output-file compare="Text">useIndex5.txt</output-file>
</test-case>
- <!--<test-case name="use-index-6" FilePath="Indexing/" Creator="Steven Jacobs">
+ <test-case name="use-index-6" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex6" date="2016-05-26"/>
<output-file compare="Text">useIndex6.txt</output-file>
- </test-case>-->
- <test-case name="use-index-7" FilePath="Indexing/" Creator="Steven Jacobs">
+ </test-case>
+ <test-case name="use-index-7" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex7" date="2016-05-26"/>
<output-file compare="Text">useIndex7.txt</output-file>
</test-case>
- <test-case name="update-index" FilePath="Indexing/" Creator="Menaka Jayawardena">
+ <test-case name="update-index" FilePath="Indexing/Partition-1/" Creator="Menaka Jayawardena">
<description>Update the existing index</description>
<query name="updateIndex" date="2016-06-24"/>
<output-file compare="Text">updateIndex.txt</output-file>
</test-case>
- <test-case name="delete-index" FilePath="Indexing/" Creator="Menaka Jayawardena">
+ <test-case name="delete-index" FilePath="Indexing/Partition-1/" Creator="Menaka Jayawardena">
<description>Delete the existing index</description>
<query name="deleteIndex" date="2016-06-24"/>
<output-file compare="Text">deleteIndex.txt</output-file>
</test-case>
+ <test-case name="show-index-2" FilePath="Indexing/Partition-1/" Creator="Menaka Jayawardena">
+ <description>Show indexes after deletion.</description>
+ <query name="showIndex" date="2016-08-18"/>
+ <output-file compare="Text">showIndex2.txt</output-file>
+ </test-case>
+ <test-case name="create-index" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+ <description>Create Lucene Index from Collection.</description>
+ <query name="createIndex" date="2016-05-26"/>
+ <output-file compare="Text">createIndex.txt</output-file>
+ </test-case>
+ <test-case name="show-index-1" FilePath="Indexing/Partition-2/" Creator="Menaka Jayawardena">
+ <description>Show indexes after creating index.</description>
+ <query name="showIndex" date="2016-08-18"/>
+ <output-file compare="Text">showIndex1.txt</output-file>
+ </test-case>
+ <test-case name="use-index-1" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex1" date="2016-05-26"/>
+ <output-file compare="Text">useIndex1.txt</output-file>
+ </test-case>
+ <test-case name="use-index-2" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex2" date="2016-05-26"/>
+ <output-file compare="Text">useIndex2.txt</output-file>
+ </test-case>
+ <test-case name="use-index-3" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex3" date="2016-05-26"/>
+ <output-file compare="Text">useIndex3.txt</output-file>
+ </test-case>
+ <test-case name="use-index-4" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex4" date="2016-05-26"/>
+ <output-file compare="Text">useIndex4.txt</output-file>
+ </test-case>
+ <test-case name="use-index-5" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex5" date="2016-05-26"/>
+ <output-file compare="Text">useIndex5.txt</output-file>
+ </test-case>
+ <test-case name="use-index-6" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex6" date="2016-05-26"/>
+ <output-file compare="Text">useIndex6.txt</output-file>
+ </test-case>
+ <test-case name="use-index-7" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex7" date="2016-05-26"/>
+ <output-file compare="Text">useIndex7.txt</output-file>
+ </test-case>
+ <test-case name="update-index" FilePath="Indexing/Partition-2/" Creator="Menaka Jayawardena">
+ <description>Update the existing index</description>
+ <query name="updateIndex" date="2016-06-24"/>
+ <output-file compare="Text">updateIndex.txt</output-file>
+ </test-case>
+ <test-case name="create-index" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+ <description>Create Lucene Index from Collection.</description>
+ <query name="createIndex" date="2016-05-26"/>
+ <output-file compare="Text">createIndex.txt</output-file>
+ </test-case>
+ <test-case name="show-index-1" FilePath="Indexing/Partition-4/" Creator="Menaka Jayawardena">
+ <description>Show indexes after creating index.</description>
+ <query name="showIndex" date="2016-08-18"/>
+ <output-file compare="Text">showIndex1.txt</output-file>
+ </test-case>
+ <test-case name="use-index-1" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex1" date="2016-05-26"/>
+ <output-file compare="Text">useIndex1.txt</output-file>
+ </test-case>
+ <test-case name="use-index-2" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex2" date="2016-05-26"/>
+ <output-file compare="Text">useIndex2.txt</output-file>
+ </test-case>
+ <test-case name="use-index-3" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex3" date="2016-05-26"/>
+ <output-file compare="Text">useIndex3.txt</output-file>
+ </test-case>
+ <test-case name="use-index-4" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex4" date="2016-05-26"/>
+ <output-file compare="Text">useIndex4.txt</output-file>
+ </test-case>
+ <test-case name="use-index-5" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex5" date="2016-05-26"/>
+ <output-file compare="Text">useIndex5.txt</output-file>
+ </test-case>
+ <test-case name="use-index-6" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex6" date="2016-05-26"/>
+ <output-file compare="Text">useIndex6.txt</output-file>
+ </test-case>
+ <test-case name="use-index-7" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex7" date="2016-05-26"/>
+ <output-file compare="Text">useIndex7.txt</output-file>
+ </test-case>
+ <test-case name="update-index" FilePath="Indexing/Partition-4/" Creator="Menaka Jayawardena">
+ <description>Update the existing index</description>
+ <query name="updateIndex" date="2016-06-24"/>
+ <output-file compare="Text">updateIndex.txt</output-file>
+ </test-case>
</test-group>
[4/4] vxquery git commit: VXQUERY-207 VXQUERY-209 Parallel Index
creation and access, and index centralization
Posted by sj...@apache.org.
VXQUERY-207 VXQUERY-209 Parallel Index creation and access, and index centralization
Fixed issue in Tests where no result file meant success
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/eb76640f
Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/eb76640f
Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/eb76640f
Branch: refs/heads/master
Commit: eb76640fc61a99b5c4bddab19afa45fe10cf0187
Parents: 6bee358
Author: Steven Glenn Jacobs <sj...@ucr.edu>
Authored: Wed Sep 7 15:44:47 2016 -0700
Committer: Steven Glenn Jacobs <sj...@ucr.edu>
Committed: Wed Sep 7 16:09:07 2016 -0700
----------------------------------------------------------------------
src/site/apt/user_indexing.apt | 202 ++++++++++++
.../java/org/apache/vxquery/cli/VXQuery.java | 1 +
.../apache/vxquery/common/VXQueryCommons.java | 43 +++
.../compiler/rewriter/RewriteRuleset.java | 2 +
.../rewriter/rules/AbstractCollectionRule.java | 31 +-
.../rewriter/rules/IntroduceCollectionRule.java | 25 +-
.../rewriter/rules/IntroduceIndexingRule.java | 58 ++++
.../rules/PushChildIntoDataScanRule.java | 11 +-
.../vxquery/functions/builtin-functions.xml | 27 +-
.../metadata/AbstractVXQueryDataSource.java | 39 +++
.../metadata/VXQueryCollectionDataSource.java | 18 +-
.../metadata/VXQueryIndexingDataSource.java | 137 ++++++++
.../VXQueryIndexingOperatorDescriptor.java | 218 +++++++++++++
.../metadata/VXQueryMetadataProvider.java | 38 ++-
...ctionFromIndexUnnestingEvaluatorFactory.java | 327 -------------------
.../IndexConstructorScalarEvaluatorFactory.java | 79 -----
.../functions/index/IndexConstructorUtil.java | 62 +---
.../index/IndexDeleteEvaluatorFactory.java | 83 -----
.../index/IndexUpdaterEvaluatorFactory.java | 84 -----
.../index/ShowIndexScalarEvaluatorFactory.java | 65 ++++
.../functions/index/VXQueryIndexReader.java | 284 ++++++++++++++++
.../indexCentralizer/IndexCentralizerUtil.java | 164 ++++++++++
.../index/indexCentralizer/IndexDirectory.java | 42 +++
.../index/indexCentralizer/IndexLocator.java | 50 +++
.../functions/index/updateIndex/Constants.java | 1 -
.../index/updateIndex/IndexUpdater.java | 72 ++--
.../index/updateIndex/MetaFileUtil.java | 149 ++++-----
.../updateIndex/XmlMetadataCollection.java | 6 +-
.../vxquery/indexing/MetaFileUtilTest.java | 36 +-
.../apache/vxquery/indexing/TestConstants.java | 8 +-
.../src/main/resources/conf/cluster_example.xml | 1 +
.../src/main/resources/conf/local.xml | 1 +
.../main/resources/scripts/cluster_actions.py | 2 +-
.../resources/scripts/cluster_information.py | 3 +
.../src/main/resources/scripts/startnc.sh | 4 +
.../apache/vxquery/xtest/TestClusterUtil.java | 12 +-
.../vxquery/xtest/AbstractXQueryTest.java | 1 +
.../Indexing/Partition-1/createIndex.txt | 1 +
.../Indexing/Partition-1/deleteIndex.txt | 1 +
.../Indexing/Partition-1/showIndex1.txt | 1 +
.../Indexing/Partition-1/showIndex2.txt | 1 +
.../Indexing/Partition-1/updateIndex.txt | 1 +
.../Indexing/Partition-1/useIndex1.txt | 2 +
.../Indexing/Partition-1/useIndex2.txt | 1 +
.../Indexing/Partition-1/useIndex3.txt | 1 +
.../Indexing/Partition-1/useIndex4.txt | 1 +
.../Indexing/Partition-1/useIndex5.txt | 3 +
.../Indexing/Partition-1/useIndex6.txt | 2 +
.../Indexing/Partition-1/useIndex7.txt | 3 +
.../Indexing/Partition-2/createIndex.txt | 2 +
.../Indexing/Partition-2/showIndex1.txt | 2 +
.../Indexing/Partition-2/updateIndex.txt | 2 +
.../Indexing/Partition-2/useIndex1.txt | 2 +
.../Indexing/Partition-2/useIndex2.txt | 1 +
.../Indexing/Partition-2/useIndex3.txt | 1 +
.../Indexing/Partition-2/useIndex4.txt | 1 +
.../Indexing/Partition-2/useIndex5.txt | 3 +
.../Indexing/Partition-2/useIndex6.txt | 2 +
.../Indexing/Partition-2/useIndex7.txt | 3 +
.../Indexing/Partition-4/createIndex.txt | 4 +
.../Indexing/Partition-4/showIndex1.txt | 6 +
.../Indexing/Partition-4/updateIndex.txt | 4 +
.../Indexing/Partition-4/useIndex1.txt | 2 +
.../Indexing/Partition-4/useIndex2.txt | 1 +
.../Indexing/Partition-4/useIndex3.txt | 1 +
.../Indexing/Partition-4/useIndex4.txt | 1 +
.../Indexing/Partition-4/useIndex5.txt | 3 +
.../Indexing/Partition-4/useIndex6.txt | 2 +
.../Indexing/Partition-4/useIndex7.txt | 3 +
.../Indexing/createIndex.txt | 1 -
.../Indexing/deleteIndex.txt | 1 -
.../Indexing/updateIndex.txt | 1 -
.../ExpectedTestResults/Indexing/useIndex1.txt | 2 -
.../ExpectedTestResults/Indexing/useIndex2.txt | 1 -
.../ExpectedTestResults/Indexing/useIndex3.txt | 1 -
.../ExpectedTestResults/Indexing/useIndex4.txt | 1 -
.../ExpectedTestResults/Indexing/useIndex5.txt | 3 -
.../ExpectedTestResults/Indexing/useIndex6.txt | 2 -
.../ExpectedTestResults/Indexing/useIndex7.txt | 3 -
.../Json/Libraries/project.txt | 13 +
.../Json/Libraries/remove_keys.txt | 12 +
.../XQuery/Indexing/Partition-1/createIndex.xq | 20 ++
.../XQuery/Indexing/Partition-1/deleteIndex.xq | 19 ++
.../XQuery/Indexing/Partition-1/showIndex.xq | 19 ++
.../XQuery/Indexing/Partition-1/updateIndex.xq | 19 ++
.../XQuery/Indexing/Partition-1/useIndex1.xq | 25 ++
.../XQuery/Indexing/Partition-1/useIndex2.xq | 24 ++
.../XQuery/Indexing/Partition-1/useIndex3.xq | 27 ++
.../XQuery/Indexing/Partition-1/useIndex4.xq | 24 ++
.../XQuery/Indexing/Partition-1/useIndex5.xq | 23 ++
.../XQuery/Indexing/Partition-1/useIndex6.xq | 23 ++
.../XQuery/Indexing/Partition-1/useIndex7.xq | 27 ++
.../XQuery/Indexing/Partition-2/createIndex.xq | 20 ++
.../XQuery/Indexing/Partition-2/showIndex.xq | 19 ++
.../XQuery/Indexing/Partition-2/updateIndex.xq | 19 ++
.../XQuery/Indexing/Partition-2/useIndex1.xq | 25 ++
.../XQuery/Indexing/Partition-2/useIndex2.xq | 24 ++
.../XQuery/Indexing/Partition-2/useIndex3.xq | 27 ++
.../XQuery/Indexing/Partition-2/useIndex4.xq | 24 ++
.../XQuery/Indexing/Partition-2/useIndex5.xq | 23 ++
.../XQuery/Indexing/Partition-2/useIndex6.xq | 23 ++
.../XQuery/Indexing/Partition-2/useIndex7.xq | 27 ++
.../XQuery/Indexing/Partition-4/createIndex.xq | 20 ++
.../XQuery/Indexing/Partition-4/showIndex.xq | 19 ++
.../XQuery/Indexing/Partition-4/updateIndex.xq | 19 ++
.../XQuery/Indexing/Partition-4/useIndex1.xq | 25 ++
.../XQuery/Indexing/Partition-4/useIndex2.xq | 24 ++
.../XQuery/Indexing/Partition-4/useIndex3.xq | 27 ++
.../XQuery/Indexing/Partition-4/useIndex4.xq | 24 ++
.../XQuery/Indexing/Partition-4/useIndex5.xq | 23 ++
.../XQuery/Indexing/Partition-4/useIndex6.xq | 23 ++
.../XQuery/Indexing/Partition-4/useIndex7.xq | 27 ++
.../Queries/XQuery/Indexing/createIndex.xq | 20 --
.../Queries/XQuery/Indexing/deleteIndex.xq | 19 --
.../Queries/XQuery/Indexing/updateIndex.xq | 19 --
.../Queries/XQuery/Indexing/useIndex1.xq | 25 --
.../Queries/XQuery/Indexing/useIndex2.xq | 24 --
.../Queries/XQuery/Indexing/useIndex3.xq | 27 --
.../Queries/XQuery/Indexing/useIndex4.xq | 24 --
.../Queries/XQuery/Indexing/useIndex5.xq | 23 --
.../Queries/XQuery/Indexing/useIndex6.xq | 23 --
.../Queries/XQuery/Indexing/useIndex7.xq | 27 --
.../src/test/resources/cat/IndexingQueries.xml | 132 +++++++-
123 files changed, 2472 insertions(+), 1094 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/src/site/apt/user_indexing.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user_indexing.apt b/src/site/apt/user_indexing.apt
new file mode 100644
index 0000000..b572231
--- /dev/null
+++ b/src/site/apt/user_indexing.apt
@@ -0,0 +1,202 @@
+~~ Licensed to the Apache Software Foundation (ASF) under one or more
+~~ contributor license agreements. See the NOTICE file distributed with
+~~ this work for additional information regarding copyright ownership.
+~~ The ASF licenses this file to You under the Apache License, Version 2.0
+~~ (the "License"); you may not use this file except in compliance with
+~~ the License. You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License.
+
+How to use Indexing Features in VXQuery.
+
+In VXQuery, all the indexes are created in user specified directory. In order to use indexing,
+you will need to set this directory in your cluster configuration file.
+
+*** Configuring VXQuery to use indexing functions.
+Add the following line to your cluster configuration (e.g. cluster.xml)
+
+--------
+ <index_directory><path_to_index_directory></index_directory>
+--------
+
+(You should create this index_directory)
+
+** Using indexing queries.
+
+VXQuery offers following indexing functionality.
+
+[[a]] Create an index for collection.
+[[b]] Use the index in executing a query.
+[[c]] Update the index.
+[[d]] Delete the index.
+[[e]] View existing indexes.
+
+*1. Scenario I - When collection is a single directory.
+In this scenario, all the XML files are stored in a single directory. (There can be sub directories)
+
+*** Creating an index for collection
+If I need to create index for xml collection stored in <path_1>/collection1,
+
+Query structure:
+--------
+build-index-on-collection("collection")
+--------
+
+You can see the index has created in a new sub-directory in the index_directory specified in local.xml
+
+Example:
+--------
+build-index-on-collection("<path_1>/collection1")
+--------
+This function takes the collection path as an argument.
+
+*** Using index in query.
+If we need to use the index and execute a query, use the following structure.
+
+------
+for $r in collection-from-index("<path1>/collection1", "/dataCollection/data")/data
+where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
+return $r
+------
+Here the index access function is,
+
+------
+collection-from-index
+------
+
+which takes two arguments, collection folder and the path element.
+
+Result
+
+------
+<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
+------
+
+*** Updating the index.
+A collection can be modified or changed by following ways.
+[[1]] Inserting new XML files.
+[[2]] Deleting files.
+[[3]] Add/ remove or modify the content of XML files.
+
+In this type of situation, the index corresponding to the modified collection must also be modified. To achieve this
+the update-index function can be used.
+
+Query structure:
+--------
+update-index("<path_to_collection>")
+--------
+
+Example:
+-------
+update-index("<path_1>/collection1")
+-------
+
+This function takes the collection which was modified.
+
+*** Deleting the index.
+If we want to delete the entire index created for a collection, the delete-index function can be used.
+This function also takes the collection path of which the index is needed to be deleted.
+
+Query structure:
+--------
+delete-index("<path_to_collection>")
+--------
+
+Example:
+-------
+delete-index("<path_11>/collection1")
+-------
+
+*2. Scenario II - When the collection is distributed.
+In this scenario, the collection is distributed among several directories. We can distribute the queries among
+partitions.
+
+*** Creating indexes for collections.
+
+Query structure:
+--------
+build-index-on-collection("<partition_1_path>|<partition_2_path>|<partition_3_path>|...|<partition_n_path>")
+--------
+
+In here the parameter contains the list of collection partitions separated by '|' character.
+
+Example:
+Consider the collection has now distributed among four directories, <path_1>/collection1, <path_2>/collection2,
+<path_3>/collection3 and <path_4>/collection4.
+
+To create indexes for all of the above collections,
+-------
+build-index-on-collection("<path_1>/collection1|<path_2>/collection2|<path_3>/collection3|<path_4>/collection4")
+-------
+
+In this case, all indexes will be created in separate sub-directories corresponding to each partition. Also note that
+ this query requires each node to have four partitions available
+
+*** Using the indexes in query.
+In this case, suppose you need to run a query on indexes of two collection partitions.
+
+Example:
+-----
+for $r in collection-from-index("<path_1>/collection1|<path-2>collection2", "/dataCollection/data")/data
+where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
+return $r
+-----
+
+The result will be taken from the indexes of both collection1 and collection2.
+
+Result:
+------
+<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
+------
+
+*** Updating the indexes.
+In cases of updating the collection files stored in several partitions, we can use this function to update the
+indexes of those directories.
+
+In this case, give a '|' separated list of collection directories.
+Query structure:
+--------
+update-index("<partition_1_path>|<partition_2_path>|<partition_3_path>|...|<partition_n_path>")
+--------
+
+Example:
+Suppose that we need to update the indexes in partition1 and partition4
+--------
+update-index("<path_1>/collection1|<path_4>/collection4")
+--------
+
+*** Deleting the indexes.
+If we want to delete indexes of collections in several partitions, we can use this function.
+Query structure:
+--------
+delete-index("<partition_1_path>|<partition_2_path>|<partition_3_path>|...|<partition_n_path>")
+--------
+
+Example:
+Suppose that we need to update the indexes in collection2 and collection3
+--------
+delete-index("<path_2>/collection2|<path_3>/collection3")
+--------
+
+** Viewing Index information.
+Suppose you need to check, what are the collections have indexes created. To do this, the show-index function can be
+used. This function takes no arguments and returns a sequence of collection paths, which an index is already created.
+If there are no indexes created for any collection, the result will be null.
+
+Suppose we have two collections, <path_1/collection1>, <path_2/collection2> have indexes created.
+Example:
+------
+show-index()
+------
+
+Result:
+------
+<path_1/collection1>
+<path_2/collection2>
+------
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
----------------------------------------------------------------------
diff --git a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
index e18332e..e0e3843 100644
--- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
+++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
@@ -305,6 +305,7 @@ public class VXQuery {
ncConfig.dataIPAddress = localAddress;
ncConfig.resultIPAddress = localAddress;
ncConfig.nodeId = "nc" + (i + 1);
+ //TODO: enable index folder as a cli option for on-the-fly indexing queries
ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString();
ncs[i] = new NodeControllerService(ncConfig);
ncs[i].start();
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java b/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
new file mode 100644
index 0000000..ceaf3c7
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
@@ -0,0 +1,43 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.common;
+
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.vxquery.functions.BuiltinFunctions;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class VXQueryCommons {
+
+ public static final Set<FunctionIdentifier> collectionFunctions = new HashSet<>();
+
+ static {
+ collectionFunctions.add(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier());
+ collectionFunctions.add(BuiltinFunctions.FN_COLLECTION_WITH_TAG_2.getFunctionIdentifier());
+ }
+
+ public static final Set<FunctionIdentifier> indexingFunctions = new HashSet<>();
+
+ static {
+ indexingFunctions.add(BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier());
+ indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier());
+ indexingFunctions.add(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier());
+ indexingFunctions.add(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java
index 205e0b2..d5909c6 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java
@@ -28,6 +28,7 @@ import org.apache.vxquery.compiler.rewriter.rules.EliminateSubplanForSingleItems
import org.apache.vxquery.compiler.rewriter.rules.EliminateUnnestAggregateSequencesRule;
import org.apache.vxquery.compiler.rewriter.rules.EliminateUnnestAggregateSubplanRule;
import org.apache.vxquery.compiler.rewriter.rules.IntroduceCollectionRule;
+import org.apache.vxquery.compiler.rewriter.rules.IntroduceIndexingRule;
import org.apache.vxquery.compiler.rewriter.rules.IntroduceTwoStepAggregateRule;
import org.apache.vxquery.compiler.rewriter.rules.PushChildIntoDataScanRule;
import org.apache.vxquery.compiler.rewriter.rules.PushFunctionsOntoEqJoinBranches;
@@ -130,6 +131,7 @@ public class RewriteRuleset {
normalization.add(new SetCollectionDataSourceRule());
normalization.add(new IntroduceCollectionRule());
normalization.add(new RemoveUnusedAssignAndAggregateRule());
+ normalization.add(new IntroduceIndexingRule());
normalization.add(new ConsolidateDescandantChild());
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java
index 74220da..eff7a6e 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java
@@ -20,6 +20,7 @@ import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.Set;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -30,8 +31,11 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
@@ -40,7 +44,6 @@ import org.apache.vxquery.compiler.algebricks.VXQueryConstantValue;
import org.apache.vxquery.compiler.rewriter.rules.util.OperatorToolbox;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
import org.apache.vxquery.datamodel.values.ValueTag;
-import org.apache.vxquery.functions.BuiltinFunctions;
import org.apache.vxquery.types.BuiltinTypeRegistry;
import org.apache.vxquery.types.Quantifier;
import org.apache.vxquery.types.SequenceType;
@@ -50,6 +53,7 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule {
final DataInputStream di = new DataInputStream(bbis);
final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
final TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
+ public static AbstractFunctionCallExpression functionCall;
/**
* Get the arguments for the collection and collection-with-tag. Return null for not a collection.
@@ -58,8 +62,7 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule {
* Logical operator
* @return collection name
*/
- protected String[] getCollectionName(Mutable<ILogicalOperator> opRef) {
-
+ protected String[] getFunctionalArguments(Mutable<ILogicalOperator> opRef, Set<FunctionIdentifier> functions) {
AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
if (op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
return null;
@@ -78,11 +81,9 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule {
if (logicalExpression.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
return null;
}
- AbstractFunctionCallExpression functionCall = (AbstractFunctionCallExpression) logicalExpression;
- if (!functionCall.getFunctionIdentifier()
- .equals(BuiltinFunctions.FN_COLLECTION_WITH_TAG_2.getFunctionIdentifier())
- && !functionCall.getFunctionIdentifier()
- .equals(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier())) {
+ functionCall = (AbstractFunctionCallExpression) logicalExpression;
+
+ if (!functions.contains(functionCall.getFunctionIdentifier())) {
return null;
}
@@ -144,6 +145,20 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule {
return null;
}
+ protected boolean setDataSourceScan(IDataSource<String> ids, Mutable<ILogicalOperator> opRef) {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ UnnestOperator unnest = (UnnestOperator) op;
+ Mutable<ILogicalOperator> opRef2 = unnest.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+ AssignOperator assign = (AssignOperator) op2;
+
+ DataSourceScanOperator opNew = new DataSourceScanOperator(assign.getVariables(), ids);
+ opNew.getInputs().addAll(assign.getInputs());
+ opRef2.setValue(opNew);
+
+ return true;
+ }
+
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
return false;
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
index 8ed8bb1..42d59aa 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
@@ -17,20 +17,15 @@
package org.apache.vxquery.compiler.rewriter.rules;
import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.vxquery.common.VXQueryCommons;
import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext;
import org.apache.vxquery.metadata.VXQueryCollectionDataSource;
import org.apache.vxquery.types.AnyItemType;
import org.apache.vxquery.types.Quantifier;
import org.apache.vxquery.types.SequenceType;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-
/**
* Find the default query plan created for collection and updated it to use
* parallelization. The rule searches for unnest followed by an assign for the
@@ -61,10 +56,11 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperat
* @author prestonc
*/
public class IntroduceCollectionRule extends AbstractCollectionRule {
+
@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context;
- String args[] = getCollectionName(opRef);
+ String args[] = getFunctionalArguments(opRef, VXQueryCommons.collectionFunctions);
if (args != null) {
String collectionName = args[0];
@@ -82,16 +78,7 @@ public class IntroduceCollectionRule extends AbstractCollectionRule {
}
// Known to be true because of collection name.
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- UnnestOperator unnest = (UnnestOperator) op;
- Mutable<ILogicalOperator> opRef2 = unnest.getInputs().get(0);
- AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
- AssignOperator assign = (AssignOperator) op2;
-
- DataSourceScanOperator opNew = new DataSourceScanOperator(assign.getVariables(), ds);
- opNew.getInputs().addAll(assign.getInputs());
- opRef2.setValue(opNew);
- return true;
+ return setDataSourceScan(ds, opRef);
}
}
return false;
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
new file mode 100644
index 0000000..5b96131
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
@@ -0,0 +1,58 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.compiler.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.vxquery.common.VXQueryCommons;
+import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext;
+import org.apache.vxquery.metadata.VXQueryIndexingDataSource;
+import org.apache.vxquery.types.AnyItemType;
+import org.apache.vxquery.types.Quantifier;
+import org.apache.vxquery.types.SequenceType;
+
+/**
+ *
+ */
+public class IntroduceIndexingRule extends AbstractCollectionRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context;
+ String args[] = getFunctionalArguments(opRef, VXQueryCommons.indexingFunctions);
+
+ if (args != null) {
+
+ String collection = args[0];
+ String elementPath = args.length > 1?args[1]:null;
+
+ // Build the new operator and update the query plan.
+ int collectionId = vxqueryContext.newCollectionId();
+ VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collection, elementPath,
+ SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR), functionCall.getFunctionIdentifier().getName());
+ if (ids != null) {
+ ids.setTotalDataSources(vxqueryContext.getTotalDataSources());
+
+ // Known to be true because of collection name.
+ return setDataSourceScan(ids, opRef);
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
index 7ffcd90..6060c19 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
@@ -34,6 +34,7 @@ import org.apache.vxquery.compiler.rewriter.rules.util.ExpressionToolbox;
import org.apache.vxquery.context.StaticContext;
import org.apache.vxquery.functions.BuiltinOperators;
import org.apache.vxquery.metadata.VXQueryCollectionDataSource;
+import org.apache.vxquery.metadata.VXQueryIndexingDataSource;
import org.apache.vxquery.metadata.VXQueryMetadataProvider;
import org.apache.vxquery.types.ElementType;
@@ -86,8 +87,16 @@ public class PushChildIntoDataScanRule extends AbstractUsedVariablesProcessingRu
DataSourceScanOperator datascan = (DataSourceScanOperator) op2;
if (!usedVariables.contains(datascan.getVariables())) {
+ VXQueryCollectionDataSource ds = null;
+ VXQueryIndexingDataSource ids = null;
+
// Find all child functions.
- VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) datascan.getDataSource();
+ try {
+ ids = (VXQueryIndexingDataSource) datascan.getDataSource();
+ } catch (ClassCastException e) {
+ ds = (VXQueryCollectionDataSource) datascan.getDataSource();
+ }
+
if (!updateDataSource(ds, unnest.getExpressionRef())) {
return false;
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
index 10ca007..688e86e 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
+++ b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
@@ -128,28 +128,25 @@
<!-- Collection operator is added during the rewrite rules phase. -->
</function>
- <!-- fn:build-index-on-collection($arg as xs:string?, $indexFolder as xs:string?) as node()* -->
+ <!-- fn:build-index-on-collection($arg as xs:string?, $indexFolder as xs:string?) as boolean -->
<function name="fn:build-index-on-collection">
<param name="collection-folder" type="xs:string?"/>
- <param name="index-folder" type="xs:string?"/>
<return type="xs:boolean"/>
- <runtime type="scalar" class="org.apache.vxquery.runtime.functions.index.IndexConstructorScalarEvaluatorFactory"/>
+ <!-- build-index-on-collection operator is added during the rewrite rules phase. -->
</function>
- <!-- fn:update-index($indexFolder as xs:string?) as node()* -->
+ <!-- fn:update-index($indexFolder as xs:string?) as boolean -->
<function name="fn:update-index">
- <param name="index-folder" type="xs:string?"/>
+ <param name="collection-folder" type="xs:string?"/>
<return type="xs:boolean"/>
- <runtime type="scalar"
- class="org.apache.vxquery.runtime.functions.index.IndexUpdaterEvaluatorFactory"/>
+ <!-- update-index operator is added during the rewrite rules phase. -->
</function>
- <!-- fn:delete-index($indexFolder as xs:string?) as node()* -->
+ <!-- fn:delete-index($indexFolder as xs:string?) as boolean -->
<function name="fn:delete-index">
- <param name="index-folder" type="xs:string?"/>
+ <param name="collection-folder" type="xs:string?"/>
<return type="xs:boolean"/>
- <runtime type="scalar"
- class="org.apache.vxquery.runtime.functions.index.IndexDeleteEvaluatorFactory"/>
+ <!-- delete-index operator is added during the rewrite rules phase. -->
</function>
<!-- fn:collection-from-index($indexfolder as xs:string?, $elementpath as xs:string?) as node()* -->
@@ -157,13 +154,19 @@
<param name="index-folder" type="xs:string?"/>
<param name="element-path" type="xs:string?"/>
<return type="node()*"/>
- <runtime type="unnesting" class="org.apache.vxquery.runtime.functions.index.CollectionFromIndexUnnestingEvaluatorFactory"/>
<property type="DocumentOrder" class="org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.InputPropertyPropagationPolicy">
<argument value="0"/>
</property>
<property type="UniqueNodes" class="org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.InputPropertyPropagationPolicy">
<argument value="0"/>
</property>
+ <!-- collection-from-index operator is added during the rewrite rules phase. -->
+ </function>
+
+ <!-- fn:show-index as node()* -->
+ <function name="fn:show-index">
+ <return type="node()*"/>
+ <runtime type="scalar" class="org.apache.vxquery.runtime.functions.index.ShowIndexScalarEvaluatorFactory"/>
</function>
<!-- fn:collection-with-tag($arg1 as xs:string?, $arg2 as xs:string?) as node()* -->
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/metadata/AbstractVXQueryDataSource.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/AbstractVXQueryDataSource.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/AbstractVXQueryDataSource.java
new file mode 100644
index 0000000..dd9898c
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/AbstractVXQueryDataSource.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.metadata;
+
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+
+import java.util.List;
+
+public abstract class AbstractVXQueryDataSource {
+ protected static final String DELIMITER = "\\|";
+ protected int dataSourceId;
+ protected String collectionName;
+ protected String[] collectionPartitions;
+ protected String elementPath;
+ protected List<Integer> childSeq;
+ protected int totalDataSources;
+ protected String tag;
+ protected String function;
+
+ protected Object[] types;
+
+ protected IDataSourcePropertiesProvider propProvider;
+
+ public abstract String getFunctionCall();
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
index 4d0e7a4..b4bc858 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
@@ -30,18 +30,7 @@ import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalProperties
import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-public class VXQueryCollectionDataSource implements IDataSource<String> {
- private static final String DELIMITER = "\\|";
- private final int dataSourceId;
- private final String collectionName;
- private String[] collectionPartitions;
- private final List<Integer> childSeq;
- private int totalDataSources;
- private String tag;
-
- private final Object[] types;
-
- private IDataSourcePropertiesProvider propProvider;
+public class VXQueryCollectionDataSource extends AbstractVXQueryDataSource implements IDataSource<String> {
private VXQueryCollectionDataSource(int id, String file, Object[] types) {
this.dataSourceId = id;
@@ -128,4 +117,9 @@ public class VXQueryCollectionDataSource implements IDataSource<String> {
public String toString() {
return "VXQueryCollectionDataSource [collectionName=" + collectionName + ", childSeq=" + childSeq + "]";
}
+
+ @Override
+ public String getFunctionCall() {
+ return function;
+ }
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
new file mode 100644
index 0000000..da75108
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
@@ -0,0 +1,137 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.metadata;
+
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import org.apache.vxquery.compiler.rewriter.rules.CollectionFileDomain;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Datasource object for indexing.
+ */
+public class VXQueryIndexingDataSource extends AbstractVXQueryDataSource implements IDataSource<String> {
+
+ protected Object[] types;
+
+ protected IDataSourcePropertiesProvider propProvider;
+ private VXQueryIndexingDataSource(int id, String collection, String elementPath, Object[] types,
+ String functionCall) {
+ this.dataSourceId = id;
+ this.collectionName = collection;
+ this.elementPath = elementPath;
+ this.function = functionCall;
+ this.collectionPartitions = collectionName.split(DELIMITER);
+
+ this.types = types;
+ final IPhysicalPropertiesVector vec = new StructuralPropertiesVector(
+ new RandomPartitioningProperty(new CollectionFileDomain(collectionName)),
+ new ArrayList<>());
+ propProvider = new IDataSourcePropertiesProvider() {
+ @Override
+ public IPhysicalPropertiesVector computePropertiesVector(List<LogicalVariable> scanVariables) {
+ return vec;
+ }
+ };
+ this.tag = null;
+ this.childSeq = new ArrayList<>();
+ }
+
+ public static VXQueryIndexingDataSource create(int id, String collection, String index, Object type, String
+ function) {
+ return new VXQueryIndexingDataSource(id, collection, index, new Object[] { type }, function);
+ }
+
+ public int getTotalDataSources() {
+ return totalDataSources;
+ }
+
+ public void setTotalDataSources(int totalDataSources) {
+ this.totalDataSources = totalDataSources;
+ }
+
+ public int getDataSourceId() {
+ return dataSourceId;
+ }
+
+ public String getElementPath() {
+ return elementPath;
+ }
+
+ public String[] getCollectionPartitions() {
+ return collectionPartitions;
+ }
+
+ public void setCollectionPartitions(String[] collectionPartitions) {
+ this.collectionPartitions = collectionPartitions;
+ }
+
+ public int getPartitionCount() {
+ return collectionPartitions.length;
+ }
+
+ public String getTag() {
+ return this.tag;
+ }
+
+ public void setTag(String tag) {
+ this.tag = tag;
+ }
+
+ @Override
+ public String getId() {
+ return collectionName;
+ }
+
+ @Override
+ public Object[] getSchemaTypes() {
+ return types;
+ }
+
+ @Override
+ public IDataSourcePropertiesProvider getPropertiesProvider() {
+ return propProvider;
+ }
+
+ @Override
+ public void computeFDs(List scanVariables, List fdList) {
+ }
+
+ @Override
+ public String toString() {
+ return "VXQueryIndexingDataSource [collectionName=" + collectionName + ", elementPath=" + elementPath + " "
+ + "function=" + function + "]";
+ }
+
+ @Override
+ public String getFunctionCall() {
+ return function;
+ }
+
+ public List<Integer> getChildSeq() {
+ return childSeq;
+ }
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
new file mode 100644
index 0000000..ae637ac
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.metadata;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.client.NodeControllerInfo;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.IFrameFieldAppender;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
+import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+import org.apache.vxquery.context.DynamicContext;
+import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.datamodel.values.XDMConstants;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.functions.BuiltinFunctions;
+import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
+import org.apache.vxquery.runtime.functions.index.VXQueryIndexReader;
+import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
+import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
+import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
+import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
+
+public class VXQueryIndexingOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ protected static final Logger LOGGER = Logger.getLogger(VXQueryCollectionOperatorDescriptor.class.getName());
+ private static final long serialVersionUID = 1L;
+ private short dataSourceId;
+ private short totalDataSources;
+ private String[] collectionPartitions;
+ private String elementPath;
+ private final String functionCall;
+
+ public VXQueryIndexingOperatorDescriptor(IOperatorDescriptorRegistry spec, VXQueryIndexingDataSource ds,
+ RecordDescriptor rDesc, String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos) {
+ super(spec, 1, 1);
+ this.functionCall = ds.getFunctionCall();
+ collectionPartitions = ds.getCollectionPartitions();
+ dataSourceId = (short) ds.getDataSourceId();
+ totalDataSources = (short) ds.getTotalDataSources();
+ recordDescriptors[0] = rDesc;
+ this.elementPath = ds.getElementPath();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ final FrameTupleAccessor fta = new FrameTupleAccessor(
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
+ final int fieldOutputCount = recordDescProvider.getOutputRecordDescriptor(getActivityId(), 0).getFieldCount();
+ final IFrame frame = new VSizeFrame(ctx);
+ final IFrameFieldAppender appender = new FrameFixedFieldTupleAppender(fieldOutputCount);
+ final short partitionId = (short) ctx.getTaskAttemptId().getTaskId().getPartition();
+ final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources);
+ final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
+ final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData();
+ final String collectionName = collectionPartitions[partition % collectionPartitions.length];
+ String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
+ IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil(
+ ctx.getIOManager().getIODevices().get(0).getPath());
+ indexCentralizerUtil.readIndexDirectory();
+
+ return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
+ @Override
+ public void open() throws HyracksDataException {
+ appender.reset(frame, true);
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ fta.reset(buffer);
+
+ IPointable result = new TaggedValuePointable();
+
+ final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
+ final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY
+ .createPointable();
+
+ final ByteBufferInputStream bbis = new ByteBufferInputStream();
+ final DataInputStream di = new DataInputStream(bbis);
+ final SequenceBuilder sb = new SequenceBuilder();
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
+
+ String indexModifiedName;
+ if (collectionModifiedName.contains("hdfs://")) {
+ throw new HyracksDataException("Indexing support for HDFS not yet implemented.");
+ } else {
+
+ if (functionCall.equals(
+ BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier().getName())) {
+ indexModifiedName = indexCentralizerUtil.putIndexForCollection(collectionModifiedName);
+ File collectionDirectory = new File(collectionModifiedName);
+
+ //check if directory is in the local file system
+ if (collectionDirectory.exists() && collectionDirectory.isDirectory()) {
+ IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
+ try {
+ indexConstructorUtil.evaluate(collectionModifiedName, indexModifiedName, result,
+ stringp, bbis, di, sb, abvs, nodeIdProvider, abvsFileNode, nodep, false,
+ nodeId);
+ XDMConstants.setTrue(result);
+ FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
+ result.getStartOffset(), result.getLength());
+ } catch (SystemException e) {
+ throw new HyracksDataException("Could not create index for collection: "
+ + collectionName + " in dir: " + indexModifiedName + " " + e.getMessage());
+ }
+ } else {
+ throw new HyracksDataException("Cannot find Collection Directory (" + nodeId + ":"
+ + collectionDirectory.getAbsolutePath() + ")");
+ }
+ } else if (functionCall
+ .equals(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier().getName())) {
+ indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+ IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs,
+ nodeIdProvider, abvsFileNode, nodep, nodeId);
+ try {
+ updater.setup();
+ updater.updateIndex();
+ updater.updateMetadataFile();
+ updater.exit();
+ XDMConstants.setTrue(result);
+ FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
+ result.getStartOffset(), result.getLength());
+ } catch (IOException | SystemException e) {
+ throw new HyracksDataException(
+ "Could not update index in " + indexModifiedName + " " + e.getMessage());
+ }
+ } else if (functionCall
+ .equals(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier().getName())) {
+ indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+ IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs,
+ nodeIdProvider, abvsFileNode, nodep, nodeId);
+ indexCentralizerUtil.deleteEntryForCollection(collectionModifiedName);
+ try {
+ updater.setup();
+ updater.deleteAllIndexes();
+ XDMConstants.setTrue(result);
+ FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
+ result.getStartOffset(), result.getLength());
+ } catch (SystemException | IOException e) {
+ throw new HyracksDataException(
+ "Could not delete index in " + indexModifiedName + " " + e.getMessage());
+ }
+
+ } else if (functionCall
+ .equals(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier().getName())) {
+ indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+ VXQueryIndexReader indexReader = new VXQueryIndexReader(ctx, indexModifiedName, elementPath);
+ try {
+ indexReader.init();
+ while (indexReader.step(result)) {
+ FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
+ result.getStartOffset(), result.getLength());
+ }
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException("Could not read index.");
+ }
+
+ } else {
+ throw new HyracksDataException("Unsupported function call (" + functionCall + ")");
+ }
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // Check if needed?
+ if (appender.getTupleCount() > 0) {
+ appender.flush(writer, true);
+ }
+ writer.close();
+ indexCentralizerUtil.writeIndexDirectory();
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
index 820c365..b7b37b9 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
@@ -88,22 +88,44 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
throws AlgebricksException {
- VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) dataSource;
+ VXQueryCollectionDataSource ds = null;
+ VXQueryIndexingDataSource ids = null;
+
+ try {
+ ids = (VXQueryIndexingDataSource) dataSource;
+ } catch (ClassCastException e) {
+ ds = (VXQueryCollectionDataSource) dataSource;
+ }
if (sourceFileMap != null) {
- final int len = ds.getPartitions().length;
+ final int len = ds != null ? ds.getPartitions().length : ids.getCollectionPartitions().length;
String[] collectionPartitions = new String[len];
for (int i = 0; i < len; ++i) {
- String partition = ds.getPartitions()[i];
+ String partition = ds != null ? ds.getPartitions()[i] : ids.getCollectionPartitions()[i];
File mapped = sourceFileMap.get(partition);
collectionPartitions[i] = mapped != null ? mapped.toString() : partition;
}
- ds.setPartitions(collectionPartitions);
+ if (ds != null) {
+ ds.setPartitions(collectionPartitions);
+ } else {
+ ids.setCollectionPartitions(collectionPartitions);
+ }
+ }
+ RecordDescriptor rDesc;
+ IOperatorDescriptor scanner;
+ AlgebricksPartitionConstraint constraint;
+
+ if (ds != null) {
+ rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]);
+ scanner = new VXQueryCollectionOperatorDescriptor(jobSpec, ds, rDesc, this.hdfsConf,
+ this.nodeControllerInfos);
+ constraint = getClusterLocations(nodeList, ds.getPartitionCount());
+ } else {
+ rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]);
+ scanner = new VXQueryIndexingOperatorDescriptor(jobSpec, ids, rDesc, this.hdfsConf,
+ this.nodeControllerInfos);
+ constraint = getClusterLocations(nodeList, ids.getPartitionCount());
}
- RecordDescriptor rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]);
- IOperatorDescriptor scanner = new VXQueryCollectionOperatorDescriptor(jobSpec, ds, rDesc, this.hdfsConf,
- this.nodeControllerInfos);
- AlgebricksPartitionConstraint constraint = getClusterLocations(nodeList, ds.getPartitionCount());
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(scanner, constraint);
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CollectionFromIndexUnnestingEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CollectionFromIndexUnnestingEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CollectionFromIndexUnnestingEvaluatorFactory.java
deleted file mode 100644
index ea78de5..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CollectionFromIndexUnnestingEvaluatorFactory.java
+++ /dev/null
@@ -1,327 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.vxquery.runtime.functions.index;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.algebricks.runtime.base.IUnnestingEvaluator;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.values.ValueTag;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.index.IndexAttributes;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentUnnestingEvaluator;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentUnnestingEvaluatorFactory;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import org.apache.vxquery.xmlparser.SAXContentHandler;
-import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
-import org.xml.sax.Attributes;
-import org.xml.sax.SAXException;
-
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class CollectionFromIndexUnnestingEvaluatorFactory extends AbstractTaggedValueArgumentUnnestingEvaluatorFactory {
- private static final long serialVersionUID = 1L;
-
- public CollectionFromIndexUnnestingEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- super(args);
- }
-
- @Override
- protected IUnnestingEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
- throws AlgebricksException {
-
- return new AbstractTaggedValueArgumentUnnestingEvaluator(args) {
-
- private ArrayBackedValueStorage nodeAbvs = new ArrayBackedValueStorage();
-
- private int indexPlace;
- private int indexLength;
- private String elementPath;
- private String indexName;
-
- private UTF8StringPointable stringIndexFolder = (UTF8StringPointable) UTF8StringPointable.FACTORY
- .createPointable();
- private UTF8StringPointable stringElementPath = (UTF8StringPointable) UTF8StringPointable.FACTORY
- .createPointable();
- private ByteBufferInputStream bbis = new ByteBufferInputStream();
- private DataInputStream di = new DataInputStream(bbis);
-
- private IndexReader reader;
- private IndexSearcher searcher;
- private Analyzer analyzer;
- private QueryParser parser;
- private ScoreDoc[] hits;
- private SAXContentHandler handler;
- private Query query;
- private Document doc;
- private List<IndexableField> fields;
-
- @Override
- public boolean step(IPointable result) throws AlgebricksException {
- /* each step will create a tuple for a single xml file
- * This is done using the parse function
- * checkoverflow is used throughout. This is because memory might not be
- * able to hold all of the results at once, so we return 1 million at
- * a time and check when we need to get more
- */
- if (indexPlace < indexLength) {
- nodeAbvs.reset();
- try {
- //TODO: now we get back the entire document
- doc = searcher.doc(hits[indexPlace].doc);
- fields = doc.getFields();
- parse(nodeAbvs);
- } catch (IOException e) {
- throw new AlgebricksException(e);
- }
- indexPlace += 1;
- result.set(nodeAbvs.getByteArray(), nodeAbvs.getStartOffset(), nodeAbvs.getLength());
- return true;
- }
- return false;
- }
-
- @Override
- protected void init(TaggedValuePointable[] args) throws SystemException {
-
- int partition = ctxview.getTaskAttemptId().getTaskId().getPartition();
- ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
- handler = new SAXContentHandler(false, nodeIdProvider, true);
-
- nodeAbvs.reset();
- indexPlace = 0;
- TaggedValuePointable tvp1 = args[0];
- TaggedValuePointable tvp2 = args[1];
-
- if (tvp1.getTag() != ValueTag.XS_STRING_TAG || tvp2.getTag() != ValueTag.XS_STRING_TAG) {
- throw new SystemException(ErrorCode.FORG0006);
- }
- tvp1.getValue(stringIndexFolder);
- tvp2.getValue(stringElementPath);
- //This whole loop is to get the string arguments, indexFolder, elementPath, and match option
- try {
- // Get the list of files.
- bbis.setByteBuffer(ByteBuffer.wrap(
- Arrays.copyOfRange(stringIndexFolder.getByteArray(), stringIndexFolder.getStartOffset(),
- stringIndexFolder.getLength() + stringIndexFolder.getStartOffset())),
- 0);
- indexName = di.readUTF();
- bbis.setByteBuffer(ByteBuffer.wrap(
- Arrays.copyOfRange(stringElementPath.getByteArray(), stringElementPath.getStartOffset(),
- stringElementPath.getLength() + stringElementPath.getStartOffset())),
- 0);
- elementPath = di.readUTF();
-
- indexPlace = 0;
-
- //Create the index reader.
- reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexName)));
- } catch (IOException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
-
- searcher = new IndexSearcher(reader);
- analyzer = new CaseSensitiveAnalyzer();
-
- parser = new CaseSensitiveQueryParser("item", analyzer);
-
- String queryString = elementPath.replaceAll("/", ".");
- queryString = "item:" + queryString + "*";
-
- int lastslash = elementPath.lastIndexOf("/");
- elementPath = elementPath.substring(0, lastslash) + ":" + elementPath.substring(lastslash + 1);
- elementPath = elementPath.replaceAll("/", ".") + ".element";
-
- TopDocs results = null;
- try {
- query = parser.parse(queryString);
-
- //TODO: Right now it only returns 1000000 results
- results = searcher.search(query, 1000000);
-
- } catch (Exception e) {
- throw new SystemException(null);
- }
-
- hits = results.scoreDocs;
- System.out.println("found: " + results.totalHits);
- indexPlace = 0;
- indexLength = hits.length;
-
- }
-
- public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException {
- try {
- handler.startDocument();
-
- for (int i = 0; i < fields.size(); i++) {
- String fieldValue = fields.get(i).stringValue();
- if (fieldValue.equals(elementPath)) {
- buildElement(abvsFileNode, i);
- }
- }
-
- handler.endDocument();
- handler.writeDocument(abvsFileNode);
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) throws SAXException {
- int whereIFinish = fieldNum;
- IndexableField field = fields.get(fieldNum);
- String contents = field.stringValue();
- String uri = "";
-
- int firstColon = contents.indexOf(':');
- int lastDot = contents.lastIndexOf('.');
- String type = contents.substring(lastDot + 1);
- String lastBit = contents.substring(firstColon + 1, lastDot);
-
- if (type.equals("textnode")) {
- char[] charContents = lastBit.toCharArray();
- handler.characters(charContents, 0, charContents.length);
-
- }
- if (type.equals("element")) {
- List<String> names = new ArrayList<String>();
- List<String> values = new ArrayList<String>();
- List<String> uris = new ArrayList<String>();
- List<String> localNames = new ArrayList<String>();
- List<String> types = new ArrayList<String>();
- List<String> qNames = new ArrayList<String>();
- whereIFinish = findAttributeChildren(whereIFinish, names, values, uris, localNames, types, qNames);
- Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
-
- handler.startElement(uri, lastBit, lastBit, atts);
-
- boolean noMoreChildren = false;
-
- while (whereIFinish + 1 < fields.size() && !noMoreChildren) {
- if (isChild(fields.get(whereIFinish + 1), field)) {
- whereIFinish = buildElement(abvsFileNode, whereIFinish + 1);
- } else {
- noMoreChildren = true;
- }
- }
-
- handler.endElement(uri, lastBit, lastBit);
-
- }
- return whereIFinish;
- }
-
- /*This function creates the attribute children for an element node
- *
- */
- int findAttributeChildren(int fieldnum, List<String> n, List<String> v, List<String> u, List<String> l,
- List<String> t, List<String> q) {
- int nextindex = fieldnum + 1;
- boolean foundattributes = false;
- if (nextindex < fields.size()) {
- IndexableField nextguy;
-
- while (nextindex < fields.size()) {
- nextguy = fields.get(nextindex);
- String contents = nextguy.stringValue();
- int firstcolon = contents.indexOf(':');
- int lastdot = contents.lastIndexOf('.');
- String lastbit = contents.substring(firstcolon + 1, lastdot);
-
- if (isDirectChildAttribute(nextguy, fields.get(fieldnum))) {
- foundattributes = true;
- n.add(lastbit);
- IndexableField nextnextguy = fields.get(nextindex + 1);
- contents = nextnextguy.stringValue();
- firstcolon = contents.indexOf(':');
- lastdot = contents.lastIndexOf('.');
- String nextlastbit = contents.substring(firstcolon + 1, lastdot);
- v.add(nextlastbit);
- u.add(lastbit);
- l.add(lastbit);
- t.add(lastbit);
- q.add(lastbit);
- } else {
- break;
- }
- nextindex += 2;
- }
- }
- if (foundattributes) {
- return nextindex - 1;
-
- } else {
- return fieldnum;
- }
- }
-
- boolean isChild(IndexableField child, IndexableField adult) {
- String childId = child.stringValue();
- String adultId = adult.stringValue();
-
- int lastDotChild = childId.lastIndexOf('.');
- int lastDotAdult = adultId.lastIndexOf('.');
-
- String childPath = childId.substring(0, lastDotChild);
- String adultPath = adultId.substring(0, lastDotAdult);
- adultPath = adultPath.replaceFirst(":", ".");
-
- return (childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + "."));
- }
-
- boolean isDirectChildAttribute(IndexableField child, IndexableField adult) {
- String childId = child.stringValue();
- String adultId = adult.stringValue();
-
- String childPath = childId.substring(0, childId.lastIndexOf('.'));
- String adultPath = adultId.substring(0, adultId.lastIndexOf('.'));
- adultPath = adultPath.replaceFirst(":", ".");
- String[] childSegments = child.stringValue().split("\\.");
-
- String childType = childSegments[childSegments.length - 1];
-
- return (childPath.startsWith(adultPath + ":") && childType.equals("attribute"));
- }
-
- };
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorScalarEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorScalarEvaluatorFactory.java
deleted file mode 100644
index cdae2cf..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorScalarEvaluatorFactory.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.XDMConstants;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
-
-import javax.xml.bind.JAXBException;
-import java.io.DataInputStream;
-
-public class IndexConstructorScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
- //Creates one Lucene doc per file
-
- private static final long serialVersionUID = 1L;
-
- public IndexConstructorScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- super(args);
- }
-
- @Override
- protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
- throws AlgebricksException {
- final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
- final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
- final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
- final ByteBufferInputStream bbis = new ByteBufferInputStream();
- final DataInputStream di = new DataInputStream(bbis);
- final SequenceBuilder sb = new SequenceBuilder();
- final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
- final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
- final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
- final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
-
- return new AbstractTaggedValueArgumentScalarEvaluator(args) {
-
- @Override
- protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
- try {
- IndexConstructorUtil
- .evaluate(args, result, stringp, bbis, di, sb, abvs, nodeIdProvider, abvsFileNode,
- nodep, false, nodeId);
- XDMConstants.setTrue(result);
- } catch (JAXBException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
- }
-
- };
- }
-}
[2/4] vxquery git commit: VXQUERY-207 VXQUERY-209 Parallel Index
creation and access, and index centralization
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/showIndex1.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/showIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/showIndex1.txt
new file mode 100644
index 0000000..31d670b
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/showIndex1.txt
@@ -0,0 +1,2 @@
+src/test/resources/TestSources/ghcnd/half_1
+src/test/resources/TestSources/ghcnd/half_2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/updateIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/updateIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/updateIndex.txt
new file mode 100644
index 0000000..0be5d98
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/updateIndex.txt
@@ -0,0 +1,2 @@
+true
+true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1.txt
new file mode 100644
index 0000000..baf9dca
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1.txt
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex2.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex2.txt
new file mode 100644
index 0000000..ef8dde4
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex2.txt
@@ -0,0 +1 @@
+<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex3.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex3.txt
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex3.txt
@@ -0,0 +1 @@
+2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex4.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex4.txt
new file mode 100644
index 0000000..f30101c
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex4.txt
@@ -0,0 +1 @@
+3.3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex5.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex5.txt
new file mode 100644
index 0000000..c84c360
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex5.txt
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex6.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex6.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex6.txt
new file mode 100644
index 0000000..9abedff
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex6.txt
@@ -0,0 +1,2 @@
+<station><id>GHCND:US000000001</id><displayName>Station 1</displayName><latitude>10.000</latitude><longitude>-10.000</longitude><elevation>1000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 1</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
+<station><id>GHCND:US000000002</id><displayName>Station 2</displayName><latitude>20.000</latitude><longitude>-20.000</longitude><elevation>2000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 2</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex7.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex7.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex7.txt
new file mode 100644
index 0000000..c84c360
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex7.txt
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/createIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/createIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/createIndex.txt
new file mode 100644
index 0000000..4b1e4bc
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/createIndex.txt
@@ -0,0 +1,4 @@
+true
+true
+true
+true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/showIndex1.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/showIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/showIndex1.txt
new file mode 100644
index 0000000..e696773
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/showIndex1.txt
@@ -0,0 +1,6 @@
+src/test/resources/TestSources/ghcnd/half_2/quarter_3
+src/test/resources/TestSources/ghcnd/half_1
+src/test/resources/TestSources/ghcnd/half_2/quarter_4
+src/test/resources/TestSources/ghcnd/half_2
+src/test/resources/TestSources/ghcnd/half_1/quarter_2
+src/test/resources/TestSources/ghcnd/half_1/quarter_1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/updateIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/updateIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/updateIndex.txt
new file mode 100644
index 0000000..4b1e4bc
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/updateIndex.txt
@@ -0,0 +1,4 @@
+true
+true
+true
+true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1.txt
new file mode 100644
index 0000000..baf9dca
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1.txt
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex2.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex2.txt
new file mode 100644
index 0000000..ef8dde4
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex2.txt
@@ -0,0 +1 @@
+<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex3.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex3.txt
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex3.txt
@@ -0,0 +1 @@
+2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex4.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex4.txt
new file mode 100644
index 0000000..f30101c
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex4.txt
@@ -0,0 +1 @@
+3.3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex5.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex5.txt
new file mode 100644
index 0000000..c84c360
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex5.txt
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex6.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex6.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex6.txt
new file mode 100644
index 0000000..9abedff
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex6.txt
@@ -0,0 +1,2 @@
+<station><id>GHCND:US000000001</id><displayName>Station 1</displayName><latitude>10.000</latitude><longitude>-10.000</longitude><elevation>1000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 1</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
+<station><id>GHCND:US000000002</id><displayName>Station 2</displayName><latitude>20.000</latitude><longitude>-20.000</longitude><elevation>2000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 2</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex7.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex7.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex7.txt
new file mode 100644
index 0000000..c84c360
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex7.txt
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/createIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/createIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/createIndex.txt
deleted file mode 100644
index f32a580..0000000
--- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/createIndex.txt
+++ /dev/null
@@ -1 +0,0 @@
-true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/deleteIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/deleteIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/deleteIndex.txt
deleted file mode 100644
index f32a580..0000000
--- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/deleteIndex.txt
+++ /dev/null
@@ -1 +0,0 @@
-true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/updateIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/updateIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/updateIndex.txt
deleted file mode 100644
index f32a580..0000000
--- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/updateIndex.txt
+++ /dev/null
@@ -1 +0,0 @@
-true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex1.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex1.txt
deleted file mode 100644
index baf9dca..0000000
--- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex1.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
-<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex2.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex2.txt
deleted file mode 100644
index ef8dde4..0000000
--- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex2.txt
+++ /dev/null
@@ -1 +0,0 @@
-<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex3.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex3.txt
deleted file mode 100644
index d8263ee..0000000
--- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex3.txt
+++ /dev/null
@@ -1 +0,0 @@
-2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex4.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex4.txt
deleted file mode 100644
index f30101c..0000000
--- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex4.txt
+++ /dev/null
@@ -1 +0,0 @@
-3.3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex5.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex5.txt
deleted file mode 100644
index c84c360..0000000
--- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex5.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
-<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
-<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex6.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex6.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex6.txt
deleted file mode 100644
index 9abedff..0000000
--- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex6.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-<station><id>GHCND:US000000001</id><displayName>Station 1</displayName><latitude>10.000</latitude><longitude>-10.000</longitude><elevation>1000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 1</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
-<station><id>GHCND:US000000002</id><displayName>Station 2</displayName><latitude>20.000</latitude><longitude>-20.000</longitude><elevation>2000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 2</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex7.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex7.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex7.txt
deleted file mode 100644
index c84c360..0000000
--- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex7.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
-<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
-<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/project.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/project.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/project.txt
new file mode 100644
index 0000000..6bb7afe
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/project.txt
@@ -0,0 +1,13 @@
+{"Captain":"Kirk","First Officer":"Spock","XQuery Evangelist":"XQuery Evangelist"}
+[1,2,3,4]
+{"Captain":"Archer","First Officer":"First Officer","XQuery Evangelist":"XQuery Evangelist"}
+true
+1
+null
+{"Captain":"Archer"}
+true
+1
+null
+true
+1
+null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/remove_keys.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/remove_keys.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/remove_keys.txt
new file mode 100644
index 0000000..41732ea
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/remove_keys.txt
@@ -0,0 +1,12 @@
+[1,2,3,4]
+true
+1
+null
+{"Engineer":"Trip"}
+true
+1
+null
+{"Captain":"Archer"}
+true
+1
+null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/createIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/createIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/createIndex.xq
new file mode 100644
index 0000000..7e8dab0
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/createIndex.xq
@@ -0,0 +1,20 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Build Lucene Index :)
+build-index-on-collection("src/test/resources/TestSources/ghcnd")
+
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/deleteIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/deleteIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/deleteIndex.xq
new file mode 100644
index 0000000..a462900
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/deleteIndex.xq
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Update Lucene Index :)
+delete-index("src/test/resources/TestSources/ghcnd")
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndex.xq
new file mode 100644
index 0000000..5ca9c2f
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndex.xq
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Show indexes after createion :)
+show-index()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/updateIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/updateIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/updateIndex.xq
new file mode 100644
index 0000000..9052693
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/updateIndex.xq
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Update Lucene Index :)
+update-index("src/test/resources/TestSources/ghcnd")
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
new file mode 100644
index 0000000..63fdda7
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003"
+ and fn:year-from-dateTime($datetime) ge 2000
+ and fn:month-from-dateTime($datetime) eq 3
+ and fn:day-from-dateTime($datetime) eq 3
+return $r
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
new file mode 100644
index 0000000..cf41536
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all reading for hurricane force wind warning or extreme wind warning. :)
+(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :)
+(: meters per second). (Wind value is in tenth of a meter per second) :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
new file mode 100644
index 0000000..5c99d9a
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
@@ -0,0 +1,27 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find the annual precipitation (PRCP) for a Seattle using the airport :)
+(: station (US000000002) for 2002. :)
+fn:sum(
+ for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+ where $r/station eq "GHCND:US000000002"
+ and $r/dataType eq "PRCP"
+ and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
+ return $r/value
+) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
new file mode 100644
index 0000000..39e5d17
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find the highest recorded temperature (TMAX) in Celsius. :)
+fn:max(
+ for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+ where $r/dataType eq "TMAX"
+ return $r/value
+) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
new file mode 100644
index 0000000..63aeca5
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
@@ -0,0 +1,23 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day :)
+(: 2002-2-2. :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
new file mode 100644
index 0000000..537e4f1
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
@@ -0,0 +1,23 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day :)
+(: 2002-2-2. :)
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station
+where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
+return $s
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
new file mode 100644
index 0000000..dd6b5f9
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
@@ -0,0 +1,27 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day :)
+(: 2002-2-2. :)
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+
+where $s/id eq $r/station
+ and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
+ and xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/createIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/createIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/createIndex.xq
new file mode 100644
index 0000000..d0ca5cc
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/createIndex.xq
@@ -0,0 +1,20 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Build Lucene Index :)
+build-index-on-collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")
+
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/showIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/showIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/showIndex.xq
new file mode 100644
index 0000000..5ca9c2f
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/showIndex.xq
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Show indexes after createion :)
+show-index()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/updateIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/updateIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/updateIndex.xq
new file mode 100644
index 0000000..6a4e6f7
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/updateIndex.xq
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Update Lucene Index :)
+update-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
new file mode 100644
index 0000000..fecb56d
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003"
+ and fn:year-from-dateTime($datetime) ge 2000
+ and fn:month-from-dateTime($datetime) eq 3
+ and fn:day-from-dateTime($datetime) eq 3
+return $r
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
new file mode 100644
index 0000000..75c7a64
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all reading for hurricane force wind warning or extreme wind warning. :)
+(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :)
+(: meters per second). (Wind value is in tenth of a meter per second) :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
new file mode 100644
index 0000000..28f7473
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
@@ -0,0 +1,27 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find the annual precipitation (PRCP) for a Seattle using the airport :)
+(: station (US000000002) for 2002. :)
+fn:sum(
+ for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+ where $r/station eq "GHCND:US000000002"
+ and $r/dataType eq "PRCP"
+ and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
+ return $r/value
+) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
new file mode 100644
index 0000000..317a141
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find the highest recorded temperature (TMAX) in Celsius. :)
+fn:max(
+ for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+ where $r/dataType eq "TMAX"
+ return $r/value
+) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
new file mode 100644
index 0000000..2deb4c3
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
@@ -0,0 +1,23 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day :)
+(: 2002-2-2. :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
new file mode 100644
index 0000000..a0ce1e9
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
@@ -0,0 +1,23 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day :)
+(: 2002-2-2. :)
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station
+where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
+return $s
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
new file mode 100644
index 0000000..b3e622c
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
@@ -0,0 +1,27 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day :)
+(: 2002-2-2. :)
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+
+where $s/id eq $r/station
+ and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
+ and xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/createIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/createIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/createIndex.xq
new file mode 100644
index 0000000..f2f0d52
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/createIndex.xq
@@ -0,0 +1,20 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Build Lucene Index :)
+build-index-on-collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")
+
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/showIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/showIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/showIndex.xq
new file mode 100644
index 0000000..5ca9c2f
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/showIndex.xq
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Show indexes after createion :)
+show-index()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/updateIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/updateIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/updateIndex.xq
new file mode 100644
index 0000000..12f5677
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/updateIndex.xq
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Update Lucene Index :)
+update-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
new file mode 100644
index 0000000..0cccbc5
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003"
+ and fn:year-from-dateTime($datetime) ge 2000
+ and fn:month-from-dateTime($datetime) eq 3
+ and fn:day-from-dateTime($datetime) eq 3
+return $r
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
new file mode 100644
index 0000000..c282e31
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all reading for hurricane force wind warning or extreme wind warning. :)
+(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :)
+(: meters per second). (Wind value is in tenth of a meter per second) :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
new file mode 100644
index 0000000..33ea1c9
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
@@ -0,0 +1,27 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find the annual precipitation (PRCP) for a Seattle using the airport :)
+(: station (US000000002) for 2002. :)
+fn:sum(
+ for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+ where $r/station eq "GHCND:US000000002"
+ and $r/dataType eq "PRCP"
+ and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
+ return $r/value
+) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
new file mode 100644
index 0000000..d213082
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find the highest recorded temperature (TMAX) in Celsius. :)
+fn:max(
+ for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+ where $r/dataType eq "TMAX"
+ return $r/value
+) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
new file mode 100644
index 0000000..1d98682
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
@@ -0,0 +1,23 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day :)
+(: 2002-2-2. :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
new file mode 100644
index 0000000..abe2184
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
@@ -0,0 +1,23 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day :)
+(: 2002-2-2. :)
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station
+where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
+return $s
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
new file mode 100644
index 0000000..7b40ca0
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
@@ -0,0 +1,27 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day :)
+(: 2002-2-2. :)
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+
+where $s/id eq $r/station
+ and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
+ and xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/createIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/createIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/createIndex.xq
deleted file mode 100644
index f34ac4c..0000000
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/createIndex.xq
+++ /dev/null
@@ -1,20 +0,0 @@
-(: Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License. :)
-
-(: Build Lucene Index :)
-build-index-on-collection( "src/test/resources/TestSources/ghcnd", "target/tmp/indexFolder")
-
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/deleteIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/deleteIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/deleteIndex.xq
deleted file mode 100644
index a409d4e..0000000
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/deleteIndex.xq
+++ /dev/null
@@ -1,19 +0,0 @@
-(: Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License. :)
-
-(: Update Lucene Index :)
-delete-index("target/tmp/indexFolder")
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/updateIndex.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/updateIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/updateIndex.xq
deleted file mode 100644
index 061f1c1..0000000
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/updateIndex.xq
+++ /dev/null
@@ -1,19 +0,0 @@
-(: Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License. :)
-
-(: Update Lucene Index :)
-update-index("target/tmp/indexFolder")
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex1.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex1.xq
deleted file mode 100644
index 1635f61..0000000
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex1.xq
+++ /dev/null
@@ -1,25 +0,0 @@
-(: Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License. :)
-
-(: Search Lucene Index :)
-for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
-let $datetime := xs:dateTime(fn:data($r/date))
-where $r/station eq "GHCND:AS000000003"
- and fn:year-from-dateTime($datetime) ge 2000
- and fn:month-from-dateTime($datetime) eq 3
- and fn:day-from-dateTime($datetime) eq 3
-return $r
\ No newline at end of file
[3/4] vxquery git commit: VXQUERY-207 VXQUERY-209 Parallel Index
creation and access, and index centralization
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
index 126ddf5..2f22ade 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
@@ -28,7 +28,6 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.ValueTag;
import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.index.IndexDocumentBuilder;
@@ -39,58 +38,31 @@ import org.apache.vxquery.xmlparser.IParser;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
import org.apache.vxquery.xmlparser.XMLParser;
-import javax.xml.bind.JAXBException;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.nio.file.Paths;
-import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
-import java.util.Arrays;
import java.util.concurrent.ConcurrentHashMap;
public class IndexConstructorUtil {
- static boolean isMetaFilePresent = false;
- static MetaFileUtil metaFileUtil;
- static ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
+ boolean isMetaFilePresent = false;
+ MetaFileUtil metaFileUtil;
+ ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
- public static void evaluate(TaggedValuePointable[] args, IPointable result, UTF8StringPointable stringp,
- ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
+ public void evaluate(String collectioFolder, String indexFolder, IPointable result, UTF8StringPointable
+ stringp, ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep,
- boolean isElementPath, String nodeId) throws SystemException, JAXBException {
- String collectionFolder;
- String indexFolder;
- TaggedValuePointable collectionTVP = args[0];
- TaggedValuePointable indexTVP = args[1];
-
- if (collectionTVP.getTag() != ValueTag.XS_STRING_TAG || indexTVP.getTag() != ValueTag.XS_STRING_TAG) {
- throw new SystemException(ErrorCode.FORG0006);
- }
+ boolean isElementPath, String nodeId) throws SystemException {
- try {
- // Get the list of files.
- collectionTVP.getValue(stringp);
- bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(),
- stringp.getLength() + stringp.getStartOffset())), 0);
- collectionFolder = di.readUTF();
-
- // Get the index folder
- indexTVP.getValue(stringp);
- bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(),
- stringp.getLength() + stringp.getStartOffset())), 0);
- indexFolder = di.readUTF();
-
- metaFileUtil = MetaFileUtil.create(indexFolder);
+ metaFileUtil = new MetaFileUtil(indexFolder);
+// metaFileUtil = .create(indexFolder);
isMetaFilePresent = metaFileUtil.isMetaFilePresent();
- metaFileUtil.setCollectionForIndex(indexFolder, collectionFolder);
+ metaFileUtil.setCollection(collectioFolder);
- } catch (IOException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
- File collectionDirectory = new File(collectionFolder);
+ File collectionDirectory = new File(collectioFolder);
if (!collectionDirectory.exists()) {
- throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist.");
+ throw new RuntimeException("The collection directory (" + collectioFolder + ") does not exist.");
}
try {
@@ -132,7 +104,7 @@ public class IndexConstructorUtil {
/*This function goes recursively one file at a time. First it turns the file into an ABVS document node, then
* it indexes that document node.
*/
- public static void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath,
+ public void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath,
TaggedValuePointable nodep, ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider,
SequenceBuilder sb, ByteBufferInputStream bbis, DataInputStream di, String nodeId)
throws SystemException, IOException {
@@ -153,11 +125,7 @@ public class IndexConstructorUtil {
xmlMetadata.setPath(file.getCanonicalPath());
xmlMetadata.setFileName(file.getName());
xmlMetadata.setLastModified(sdf.format(file.lastModified()));
- try {
- xmlMetadata.setMd5(metaFileUtil.generateMD5(file));
- } catch (NoSuchAlgorithmException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
+ xmlMetadata.setMd5(metaFileUtil.generateMD5(file));
metadataMap.put(file.getCanonicalPath(), xmlMetadata);
}
@@ -168,14 +136,14 @@ public class IndexConstructorUtil {
}
}
- public static boolean readableXmlFile(String path) {
+ public boolean readableXmlFile(String path) {
return (path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz"));
}
/**
* Separated from create index method so that it could be used as a helper function in IndexUpdater
*/
- public static IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep,
+ public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep,
ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, ByteBufferInputStream bbis,
DataInputStream di, String nodeId) throws IOException {
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java
deleted file mode 100644
index e713b20..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.XDMConstants;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
-
-import javax.xml.bind.JAXBException;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-
-/**
- * Delete the index of a given index directory
- */
-public class IndexDeleteEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
- public IndexDeleteEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- super(args);
- }
-
- @Override
- protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
- throws AlgebricksException {
- final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
- final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
- final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
- final ByteBufferInputStream bbis = new ByteBufferInputStream();
- final DataInputStream di = new DataInputStream(bbis);
- final SequenceBuilder sb = new SequenceBuilder();
- final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
- final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
- final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
- final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
-
- return new AbstractTaggedValueArgumentScalarEvaluator(args) {
-
- @Override
- protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
- IndexUpdater updater = new IndexUpdater(args, result, stringp, bbis, di, sb, abvs, nodeIdProvider,
- abvsFileNode, nodep, nodeId);
- try {
- updater.setup();
- updater.deleteAllIndexes();
- XDMConstants.setTrue(result);
- } catch (IOException | NoSuchAlgorithmException | JAXBException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
-
- }
-
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java
deleted file mode 100644
index 3fbc279..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.XDMConstants;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
-
-import javax.xml.bind.JAXBException;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-
-/**
- * Update the index of collection
- */
-public class IndexUpdaterEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
- public IndexUpdaterEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- super(args);
- }
-
- @Override
- protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
- throws AlgebricksException {
- final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
- final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
- final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
- final ByteBufferInputStream bbis = new ByteBufferInputStream();
- final DataInputStream di = new DataInputStream(bbis);
- final SequenceBuilder sb = new SequenceBuilder();
- final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
- final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
- final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
- final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
-
- return new AbstractTaggedValueArgumentScalarEvaluator(args) {
-
- @Override
- protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
- IndexUpdater updater = new IndexUpdater(args, result, stringp, bbis, di, sb, abvs, nodeIdProvider,
- abvsFileNode, nodep, nodeId);
- try {
- updater.setup();
- updater.updateIndex();
- updater.updateMetadataFile();
- updater.exit();
- XDMConstants.setTrue(result);
- } catch (IOException | NoSuchAlgorithmException | JAXBException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
- }
-
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
new file mode 100644
index 0000000..9b72a34
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
@@ -0,0 +1,65 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index;
+
+import java.io.IOException;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
+import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
+import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
+
+public class ShowIndexScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
+ public ShowIndexScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+ super(args);
+ }
+
+ @Override
+ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
+ throws AlgebricksException {
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ final SequenceBuilder sb = new SequenceBuilder();
+
+ return new AbstractTaggedValueArgumentScalarEvaluator(args) {
+ @Override
+ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
+ try {
+ abvs.reset();
+ sb.reset(abvs);
+ IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil(
+ ctx.getIOManager().getIODevices().get(0).getPath());
+ indexCentralizerUtil.readIndexDirectory();
+ indexCentralizerUtil.getAllCollections(sb);
+ sb.finish();
+ result.set(abvs);
+ } catch (IOException e) {
+ throw new SystemException(ErrorCode.SYSE0001, e);
+ }
+
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
new file mode 100644
index 0000000..8750849
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
@@ -0,0 +1,284 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.index.IndexAttributes;
+import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
+import org.apache.vxquery.xmlparser.SAXContentHandler;
+import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+public class VXQueryIndexReader {
+
+ private ArrayBackedValueStorage nodeAbvs = new ArrayBackedValueStorage();
+
+ private int indexPlace;
+ private int indexLength;
+ private String elementPath;
+ private String indexName;
+
+ private ByteBufferInputStream bbis = new ByteBufferInputStream();
+ private DataInputStream di = new DataInputStream(bbis);
+
+ private IndexReader reader;
+ private IndexSearcher searcher;
+ private QueryParser parser;
+ private ScoreDoc[] hits;
+ private SAXContentHandler handler;
+ private Query query;
+ private Document doc;
+ private List<IndexableField> fields;
+ private IHyracksTaskContext ctx;
+
+ public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, String elementPath) {
+ this.ctx = context;
+ this.indexName = indexPath;
+ this.elementPath = elementPath;
+ }
+
+ public boolean step(IPointable result) throws AlgebricksException {
+ /*each step will create a tuple for a single xml file
+ * This is done using the parse function
+ * checkoverflow is used throughout. This is because memory might not be
+ * able to hold all of the results at once, so we return 1 million at
+ * a time and check when we need to get more
+ */
+ if (indexPlace < indexLength) {
+ nodeAbvs.reset();
+ try {
+ //TODO: now we get back the entire document
+ doc = searcher.doc(hits[indexPlace].doc);
+ fields = doc.getFields();
+ parse(nodeAbvs);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ indexPlace += 1;
+ result.set(nodeAbvs.getByteArray(), nodeAbvs.getStartOffset(), nodeAbvs.getLength());
+ return true;
+ }
+ return false;
+ }
+
+ public void init() throws SystemException {
+
+ int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
+ ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
+ handler = new SAXContentHandler(false, nodeIdProvider, true);
+
+ nodeAbvs.reset();
+ indexPlace = 0;
+
+ try {
+ indexPlace = 0;
+
+ //Create the index reader.
+ reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexName)));
+ } catch (IOException e) {
+ throw new SystemException(ErrorCode.SYSE0001, e);
+ }
+
+ searcher = new IndexSearcher(reader);
+ Analyzer analyzer = new CaseSensitiveAnalyzer();
+
+ parser = new CaseSensitiveQueryParser("item", analyzer);
+
+ String queryString = elementPath.replaceAll("/", ".");
+ queryString = "item:" + queryString + "*";
+
+ int lastslash = elementPath.lastIndexOf("/");
+ elementPath = elementPath.substring(0, lastslash) + ":" + elementPath.substring(lastslash + 1);
+ elementPath = elementPath.replaceAll("/", ".") + ".element";
+
+ TopDocs results = null;
+ try {
+ query = parser.parse(queryString);
+
+ //TODO: Right now it only returns 1000000 results
+ results = searcher.search(query, 1000000);
+
+ } catch (Exception e) {
+ throw new SystemException(null);
+ }
+
+ hits = results.scoreDocs;
+ System.out.println("found: " + results.totalHits);
+ indexPlace = 0;
+ indexLength = hits.length;
+
+ }
+
+ public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException {
+ try {
+ handler.startDocument();
+
+ for (int i = 0; i < fields.size(); i++) {
+ String fieldValue = fields.get(i).stringValue();
+ if (fieldValue.equals(elementPath)) {
+ buildElement(abvsFileNode, i);
+ }
+ }
+
+ handler.endDocument();
+ handler.writeDocument(abvsFileNode);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) throws SAXException {
+ int whereIFinish = fieldNum;
+ IndexableField field = fields.get(fieldNum);
+ String contents = field.stringValue();
+ String uri = "";
+
+ int firstColon = contents.indexOf(':');
+ int lastDot = contents.lastIndexOf('.');
+ String type = contents.substring(lastDot + 1);
+ String lastBit = contents.substring(firstColon + 1, lastDot);
+
+ if (type.equals("textnode")) {
+ char[] charContents = lastBit.toCharArray();
+ handler.characters(charContents, 0, charContents.length);
+
+ }
+ if (type.equals("element")) {
+ List<String> names = new ArrayList<String>();
+ List<String> values = new ArrayList<String>();
+ List<String> uris = new ArrayList<String>();
+ List<String> localNames = new ArrayList<String>();
+ List<String> types = new ArrayList<String>();
+ List<String> qNames = new ArrayList<String>();
+ whereIFinish = findAttributeChildren(whereIFinish, names, values, uris, localNames, types, qNames);
+ Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
+
+ handler.startElement(uri, lastBit, lastBit, atts);
+
+ boolean noMoreChildren = false;
+
+ while (whereIFinish + 1 < fields.size() && !noMoreChildren) {
+ if (isChild(fields.get(whereIFinish + 1), field)) {
+ whereIFinish = buildElement(abvsFileNode, whereIFinish + 1);
+ } else {
+ noMoreChildren = true;
+ }
+ }
+
+ handler.endElement(uri, lastBit, lastBit);
+
+ }
+ return whereIFinish;
+ }
+
+ /*This function creates the attribute children for an element node
+ *
+ */
+ int findAttributeChildren(int fieldnum, List<String> n, List<String> v, List<String> u, List<String> l,
+ List<String> t, List<String> q) {
+ int nextindex = fieldnum + 1;
+ boolean foundattributes = false;
+ if (nextindex < fields.size()) {
+ IndexableField nextguy;
+
+ while (nextindex < fields.size()) {
+ nextguy = fields.get(nextindex);
+ String contents = nextguy.stringValue();
+ int firstcolon = contents.indexOf(':');
+ int lastdot = contents.lastIndexOf('.');
+ String lastbit = contents.substring(firstcolon + 1, lastdot);
+
+ if (isDirectChildAttribute(nextguy, fields.get(fieldnum))) {
+ foundattributes = true;
+ n.add(lastbit);
+ IndexableField nextnextguy = fields.get(nextindex + 1);
+ contents = nextnextguy.stringValue();
+ firstcolon = contents.indexOf(':');
+ lastdot = contents.lastIndexOf('.');
+ String nextlastbit = contents.substring(firstcolon + 1, lastdot);
+ v.add(nextlastbit);
+ u.add(lastbit);
+ l.add(lastbit);
+ t.add(lastbit);
+ q.add(lastbit);
+ } else {
+ break;
+ }
+ nextindex += 2;
+ }
+ }
+ if (foundattributes) {
+ return nextindex - 1;
+
+ } else {
+ return fieldnum;
+ }
+ }
+
+ boolean isChild(IndexableField child, IndexableField adult) {
+ String childId = child.stringValue();
+ String adultId = adult.stringValue();
+
+ int lastDotChild = childId.lastIndexOf('.');
+ int lastDotAdult = adultId.lastIndexOf('.');
+
+ String childPath = childId.substring(0, lastDotChild);
+ String adultPath = adultId.substring(0, lastDotAdult);
+ adultPath = adultPath.replaceFirst(":", ".");
+
+ return (childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + "."));
+ }
+
+ boolean isDirectChildAttribute(IndexableField child, IndexableField adult) {
+ String childId = child.stringValue();
+ String adultId = adult.stringValue();
+
+ String childPath = childId.substring(0, childId.lastIndexOf('.'));
+ String adultPath = adultId.substring(0, adultId.lastIndexOf('.'));
+ adultPath = adultPath.replaceFirst(":", ".");
+ String[] childSegments = child.stringValue().split("\\.");
+
+ String childType = childSegments[childSegments.length - 1];
+
+ return (childPath.startsWith(adultPath + ":") && childType.equals("attribute"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
new file mode 100644
index 0000000..38dcc2a
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+
+import java.io.DataOutput;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.vxquery.datamodel.builders.atomic.StringValueBuilder;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.datamodel.values.ValueTag;
+
+/**
+ * Class for maintaining the centralized index information file.
+ * Index centralization procedure.
+ * User can specify the collection directory in VXQuery.java, ncConfig.ioDevices = <index_directory>.
+ * Then all the indexes will be created in that particular directory in sub-folders corresponding to collections.
+ * There will be a single xml file, located in the directory specified in local.xml, which contains all information
+ * about the existing indexes.
+ * This class can be used to read, add, delete, modify the entries and write the file back to the disk.
+ */
+public class IndexCentralizerUtil {
+
+ private final String FILE_NAME = "VXQuery-Index-Directory.xml";
+ private final List<String> collections = new ArrayList<>();
+ private final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil");
+ private File XML_FILE;
+ private String INDEX_LOCATION;
+ private static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>();
+
+ public IndexCentralizerUtil(File index) {
+ this.INDEX_LOCATION = index.getPath();
+ if (!index.exists()) {
+ try {
+ FileUtils.forceMkdir(index);
+ } catch (IOException e) {
+ LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + INDEX_LOCATION + " " + e);
+ }
+ }
+ XML_FILE = new File(index.getPath() + "/" + FILE_NAME);
+ }
+
+ /**
+ * Get the index directory containing index of the given collection
+ *
+ * @param collection : Collection folder
+ * @return Index folder.
+ */
+ public String getIndexForCollection(String collection) {
+ return indexCollectionMap.get(collection).getIndex();
+ }
+
+ /**
+ * Put the index location corresponding to given collection.
+ * Index location is created by using the last 100 characters of collection.
+ *
+ * @param collection : Collection directory
+ */
+ public String putIndexForCollection(String collection) {
+ int length = collection.replaceAll("/", "").length();
+ String index = collection.replaceAll("/", "");
+ index = INDEX_LOCATION + "/" + (length > 100 ? index.substring(length - 100) : index);
+ IndexLocator il = new IndexLocator();
+ il.setCollection(collection);
+ il.setIndex(index);
+ if (indexCollectionMap.get(collection) != null) {
+ return index;
+ }
+ indexCollectionMap.put(collection, il);
+ return index;
+ }
+
+ /**
+ * Remove the entry for given collection directory.
+ *
+ * @param collection : Collection directory
+ */
+ public void deleteEntryForCollection(String collection) {
+ indexCollectionMap.remove(collection);
+ }
+
+ /**
+ * Prints all collections which have an index created.
+ *
+ * @throws IOException
+ */
+ public void getAllCollections(SequenceBuilder sb) throws IOException {
+ for (String s : collections) {
+ StringValueBuilder svb = new StringValueBuilder();
+ ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ DataOutput output = abvs.getDataOutput();
+ output.write(ValueTag.XS_STRING_TAG);
+ svb.write(s, output);
+ sb.addItem(abvs);
+ }
+ }
+
+ /**
+ * Read the collection, index directory file and populate the HashMap.
+ */
+ public void readIndexDirectory() {
+ if (this.XML_FILE.exists()) {
+ try {
+ JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class);
+ Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
+ IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(this.XML_FILE);
+
+ for (IndexLocator il : indexDirectory.getDirectory()) {
+ indexCollectionMap.put(il.getCollection(), il);
+ this.collections.add(il.getCollection());
+ }
+ } catch (JAXBException e) {
+ LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e);
+ }
+ }
+
+ }
+
+ /**
+ * Write back the contents of the HashMap to the file.
+ */
+ public void writeIndexDirectory() {
+ IndexDirectory id = new IndexDirectory();
+ List<IndexLocator> indexLocators = new ArrayList<>(indexCollectionMap.values());
+ id.setDirectory(indexLocators);
+ try {
+ FileOutputStream fileOutputStream = new FileOutputStream(this.XML_FILE);
+ JAXBContext context = JAXBContext.newInstance(IndexDirectory.class);
+ Marshaller jaxbMarshaller = context.createMarshaller();
+ jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+ jaxbMarshaller.marshal(id, fileOutputStream);
+ } catch (JAXBException | FileNotFoundException e) {
+ LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
new file mode 100644
index 0000000..54d9ad9
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+@XmlRootElement(name = "indexes")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class IndexDirectory implements Serializable{
+
+ @XmlElement(name = "index", type = IndexLocator.class)
+ private List<IndexLocator> directory = new ArrayList<>();
+
+ public List<IndexLocator> getDirectory() {
+ return directory;
+ }
+
+
+ public void setDirectory(List<IndexLocator> directory) {
+ this.directory = directory;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
new file mode 100644
index 0000000..1a33c8b
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlRootElement(name = "Entry")
+public class IndexLocator implements Serializable{
+
+ @XmlAttribute
+ private String collection;
+
+ @XmlAttribute
+ private String index;
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public void setCollection(String collection) {
+ this.collection = collection;
+ }
+
+ public String getIndex() {
+ return index;
+ }
+
+ public void setIndex(String index) {
+ this.index = index;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
index 9aebfb4..2a45747 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
@@ -22,5 +22,4 @@ package org.apache.vxquery.runtime.functions.index.updateIndex;
public class Constants {
public static String FIELD_PATH = "path";
public static String META_FILE_NAME = "vxquery_index.xml";
- public static String COLLECTION_ENTRY = "collection";
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
index 4588282..ba7cd88 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
@@ -29,7 +29,6 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.ValueTag;
import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.index.IndexDocumentBuilder;
@@ -37,16 +36,12 @@ import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer;
import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import javax.xml.bind.JAXBException;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -57,9 +52,7 @@ import java.util.concurrent.ConcurrentHashMap;
public class IndexUpdater {
private MetaFileUtil metaFileUtil;
private ConcurrentHashMap<String, XmlMetadata> metadataMap;
- private TaggedValuePointable[] args;
private IPointable result;
- private UTF8StringPointable stringp;
private ByteBufferInputStream bbis;
private DataInputStream di;
private SequenceBuilder sb;
@@ -71,19 +64,17 @@ public class IndexUpdater {
private IndexWriter indexWriter;
private Set<String> pathsFromFileList;
private String collectionFolder;
- private XmlMetadata collectionMetadata;
private String indexFolder;
private Logger LOGGER = Logger.getLogger("Index Updater");
private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
+ private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
- //TODO : Implement for paralleizing
- public IndexUpdater(TaggedValuePointable[] args, IPointable result, UTF8StringPointable stringp,
+ public IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp,
ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep,
String nodeId) {
- this.args = args;
+ this.indexFolder = indexFolder;
this.result = result;
- this.stringp = stringp;
this.bbis = bbis;
this.di = di;
this.sb = sb;
@@ -100,35 +91,17 @@ public class IndexUpdater {
*
* @throws SystemException
* @throws IOException
- * @throws NoSuchAlgorithmException
*/
- public void setup() throws SystemException, IOException, NoSuchAlgorithmException, JAXBException {
+ public void setup() throws SystemException, IOException {
- TaggedValuePointable indexTVP = args[0];
+ // Read the metadata file and load the metadata map into memory.
+ metaFileUtil = new MetaFileUtil(indexFolder);
+ metaFileUtil.readMetadataFile();
+ metadataMap = metaFileUtil.getMetadata();
- if (indexTVP.getTag() != ValueTag.XS_STRING_TAG) {
- throw new SystemException(ErrorCode.FORG0006);
- }
-
- try {
- // Get the index folder
- indexTVP.getValue(stringp);
- bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(),
- stringp.getLength() + stringp.getStartOffset())), 0);
- indexFolder = di.readUTF();
-
- // Read the metadata file and load the metadata map into memory.
- metaFileUtil = MetaFileUtil.create(indexFolder);
- metaFileUtil.readMetadataFile();
- metadataMap = metaFileUtil.getMetadata(indexFolder);
-
- // Retrieve the collection folder path.
- // Remove the entry for ease of the next steps.
- collectionFolder = metaFileUtil.getCollection(indexFolder);
-
- } catch (IOException | ClassNotFoundException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
+ // Retrieve the collection folder path.
+ // Remove the entry for ease of the next steps.
+ collectionFolder = metaFileUtil.getCollection();
abvs.reset();
sb.reset(abvs);
@@ -142,9 +115,8 @@ public class IndexUpdater {
* Wrapper for update index function.
*
* @throws IOException
- * @throws NoSuchAlgorithmException
*/
- public void updateIndex() throws IOException, NoSuchAlgorithmException {
+ public void updateIndex() throws IOException {
File collectionDirectory = new File(collectionFolder);
if (!collectionDirectory.exists()) {
throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist.");
@@ -155,6 +127,7 @@ public class IndexUpdater {
//Detect deleted files and execute the delete index process.
deleteIndexOfDeletedFiles(metadataMap.keySet(), pathsFromFileList);
+ updateMetadataFile();
}
/**
@@ -176,7 +149,7 @@ public class IndexUpdater {
*
* @throws IOException
*/
- public synchronized void updateMetadataFile() throws IOException, JAXBException {
+ public synchronized void updateMetadataFile() throws IOException {
//Write the updated metadata to the file.
metaFileUtil.updateMetadataMap(metadataMap, indexFolder);
metaFileUtil.writeMetadataToFile();
@@ -188,14 +161,14 @@ public class IndexUpdater {
*
* @param collection : Collection folder path
*/
- private void updateIndex(File collection) throws IOException, NoSuchAlgorithmException {
+ private void updateIndex(File collection) throws IOException {
File[] list = collection.listFiles();
assert list != null;
for (File file : list) {
pathsFromFileList.add(file.getCanonicalPath());
- if (IndexConstructorUtil.readableXmlFile(file.getCanonicalPath())) {
+ if (indexConstructorUtil.readableXmlFile(file.getCanonicalPath())) {
XmlMetadata data = metadataMap.get(file.getCanonicalPath());
String md5 = metaFileUtil.generateMD5(file);
@@ -212,7 +185,7 @@ public class IndexUpdater {
//Update index corresponding to the xml file.
indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath()));
- indexDocumentBuilder = IndexConstructorUtil
+ indexDocumentBuilder = indexConstructorUtil
.getIndexBuilder(file, indexWriter, nodep, abvsFileNode, nodeIdProvider, bbis, di,
nodeId);
indexDocumentBuilder.printStart();
@@ -230,7 +203,7 @@ public class IndexUpdater {
// In this case, the xml file has not added to the index. (It is a newly added file)
// Therefore generate a new index for this file and add it to the existing index.
- indexDocumentBuilder = IndexConstructorUtil
+ indexDocumentBuilder = indexConstructorUtil
.getIndexBuilder(file, indexWriter, nodep, abvsFileNode, nodeIdProvider, bbis, di, nodeId);
indexDocumentBuilder.printStart();
@@ -254,15 +227,14 @@ public class IndexUpdater {
* @param metadata : Existing metadata object
* @return : XML metadata object with updated fields.
* @throws IOException
- * @throws NoSuchAlgorithmException
*/
- private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException, NoSuchAlgorithmException {
+ private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException {
- if (metadata == null)
+ if (metadata == null) {
metadata = new XmlMetadata();
-
- metadata.setPath(file.getCanonicalPath());
+ }
metadata.setFileName(file.getName());
+ metadata.setPath(file.getCanonicalPath());
metadata.setMd5(metaFileUtil.generateMD5(file));
metadata.setLastModified(sdf.format(file.lastModified()));
return metadata;
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
index 53b02df..dd099b5 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
@@ -19,7 +19,11 @@ package org.apache.vxquery.runtime.functions.index.updateIndex;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import javax.xml.bind.*;
+import javax.xml.bind.DatatypeConverter;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -40,17 +44,14 @@ public class MetaFileUtil {
private File metaFile;
private Logger LOGGER = Logger.getLogger("MetadataFileUtil");
- private Map<String, ConcurrentHashMap<String, XmlMetadata>> indexes = new ConcurrentHashMap<>();
- private Map<String, String> indexToCollection = new ConcurrentHashMap<>();
+ private String index;
+ private String collection;
+ private ConcurrentHashMap<String, XmlMetadata> indexMap = new ConcurrentHashMap<>();
- private MetaFileUtil(String indexFolder) {
+ public MetaFileUtil(String indexFolder) {
this.metaFile = new File(indexFolder + "/" + Constants.META_FILE_NAME);
}
- public static MetaFileUtil create(String indexFolder) {
- return new MetaFileUtil(indexFolder);
- }
-
/**
* Checks for existing metadata file.
*
@@ -66,88 +67,74 @@ public class MetaFileUtil {
* Otherwise insert new.
* @param metadataMap : Set of XmlMetaData objects.
* @param index : The path to index location.
- * @throws IOException
*/
- public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) throws
- IOException, JAXBException {
-
- if (this.indexes.get(index) == null) {
- this.indexes.put(index, metadataMap);
- } else {
- this.indexes.replace(index, metadataMap);
- }
+ public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) {
+ this.indexMap = metadataMap;
+ this.index = index;
}
/**
* Method to get the set of xml metadata for a given collection
*
- * @param index : The collection from which the metadata should be read.
- * @return : Map containing the set of XmlMetadata objects.
- * @throws IOException
- * @throws ClassNotFoundException
+ * @return : Map containing the set of XmlMetadata objects.\
*/
- public ConcurrentHashMap<String, XmlMetadata> getMetadata(String index)
- throws IOException, ClassNotFoundException, JAXBException {
-
- return this.indexes.get(index);
+ public ConcurrentHashMap<String, XmlMetadata> getMetadata() {
+ return this.indexMap;
}
/**
* Read the metadata file and create an in-memory map containing collection paths and xml files.
- * @throws JAXBException
*/
- public void readMetadataFile() throws JAXBException {
- JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
- Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
- VXQueryIndex indexes = (VXQueryIndex) jaxbUnmarshaller.unmarshal(metaFile);
-
- List<XmlMetadataCollection> list = indexes.getIndex();
-
-
- for (XmlMetadataCollection collection : list) {
- String indexPath = collection.getIndexLocation();
- ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
- List<XmlMetadata> metadata = collection.getMetadataList();
+ public void readMetadataFile() {
+ try {
+ JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
+ Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
+ XmlMetadataCollection indexes = (XmlMetadataCollection) jaxbUnmarshaller.unmarshal(metaFile);
- this.indexToCollection.put(indexPath, collection.getCollection());
+ this.collection = indexes.getCollection();
+ this.index = indexes.getIndexLocation();
- for (XmlMetadata mData : metadata) {
- metadataMap.put(mData.getPath(), mData);
+ for (XmlMetadata metadata : indexes.getMetadataList()) {
+ this.indexMap.put(index, metadata);
+ }
+ } catch (JAXBException e) {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e);
}
- this.indexes.put(indexPath, metadataMap);
}
}
/**
* Write the content of the ConcurrentHashMap to the xml metadata file.
- * @throws FileNotFoundException
- * @throws JAXBException
*/
- public void writeMetadataToFile() throws FileNotFoundException, JAXBException {
- VXQueryIndex index = new VXQueryIndex();
- List<XmlMetadataCollection> xmlMetadataCollections = new ArrayList<>();
-
- for (Map.Entry<String, ConcurrentHashMap<String, XmlMetadata>> entry : indexes.entrySet()) {
- XmlMetadataCollection metadataCollection = new XmlMetadataCollection();
- List<XmlMetadata> metadataList = new ArrayList<>();
- metadataCollection.setIndexLocation(entry.getKey());
- metadataCollection.setCollection(indexToCollection.get(entry.getKey()));
- metadataList.addAll(entry.getValue().values());
- metadataCollection.setMetadataList(metadataList);
- xmlMetadataCollections.add(metadataCollection);
- }
- index.setIndex(xmlMetadataCollections);
+ public void writeMetadataToFile() {
+ XmlMetadataCollection collection = new XmlMetadataCollection();
+ List<XmlMetadata> metadataList = new ArrayList<>();
+ for (Map.Entry<String, XmlMetadata> entry : this.indexMap.entrySet()) {
+ metadataList.add(entry.getValue());
+ }
- FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile);
- JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
- Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
- jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
- jaxbMarshaller.marshal(index, fileOutputStream);
+ collection.setMetadataList(metadataList);
+ collection.setCollection(this.collection);
+ collection.setIndexLocation(this.index);
+ try{
+ FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile);
+ JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
+ Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
+ jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+ jaxbMarshaller.marshal(collection, fileOutputStream);
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!");
+ }
+ } catch (JAXBException | FileNotFoundException e) {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e);
+ }
+ }
- if (LOGGER.isDebugEnabled())
- LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!");
}
@@ -157,14 +144,20 @@ public class MetaFileUtil {
*
* @param file : File which the checksum should be generated.
* @return : Checksum String
- * @throws NoSuchAlgorithmException
* @throws IOException
*/
- public String generateMD5(File file) throws NoSuchAlgorithmException, IOException {
- MessageDigest md = MessageDigest.getInstance("MD5");
- md.update(Files.readAllBytes(file.toPath()));
- byte[] md5 = md.digest();
- return DatatypeConverter.printHexBinary(md5);
+ public String generateMD5(File file) throws IOException {
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update(Files.readAllBytes(file.toPath()));
+ byte[] md5 = md.digest();
+ return DatatypeConverter.printHexBinary(md5);
+ } catch (NoSuchAlgorithmException e) {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.log(Level.ERROR, "No Such Algorithm Error " + e.getMessage());
+ }
+ return null;
+ }
}
/**
@@ -189,21 +182,17 @@ public class MetaFileUtil {
/**
* Get the collection for a given index location.
- * @param index : path to index
- * @return
+ * @return collection folder for a given index.
*/
- public String getCollection(String index) {
- return this.indexToCollection.get(index);
+ public String getCollection() {
+ return this.collection;
}
/**
* Set the entry for given index and collection.
- * @param index : path to index
* @param collection : path to corresponding collection
*/
- public void setCollectionForIndex(String index, String collection) {
- if (this.indexToCollection.get(index)==null) {
- this.indexToCollection.put(index, collection);
- }
+ public void setCollection(String collection) {
+ this.collection = collection;
}
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
index 270fb8d..1f5c3e9 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
@@ -16,7 +16,11 @@
*/
package org.apache.vxquery.runtime.functions.index.updateIndex;
-import javax.xml.bind.annotation.*;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
import java.util.List;
/**
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
index 6fa92e1..543719d 100644
--- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
+++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
@@ -14,7 +14,14 @@
*/
package org.apache.vxquery.indexing;
-import junit.framework.Assert;
+import java.io.File;
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.xml.bind.JAXBException;
+
import org.apache.commons.io.FileUtils;
import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil;
import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
@@ -24,12 +31,7 @@ import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
-import javax.xml.bind.JAXBException;
-import java.io.File;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
+import junit.framework.Assert;
/**
* Test cases for testing MetaFileUtil functions.
@@ -49,7 +51,7 @@ public class MetaFileUtilTest {
@BeforeClass
public static void setup() {
new File(TestConstants.INDEX_DIR).mkdir();
- metaFileUtil = MetaFileUtil.create(TestConstants.INDEX_DIR);
+ metaFileUtil = new MetaFileUtil(TestConstants.INDEX_DIR);
initialMap = TestConstants.getInitialMap();
modifiedMap = TestConstants.getModifiedMap();
}
@@ -58,7 +60,7 @@ public class MetaFileUtilTest {
* Test case for generating MD5 string for an XML file.
*/
@Test
- public void step1_testGenerateMD5ForXML() throws IOException, NoSuchAlgorithmException {
+ public void step1_testGenerateMD5ForXML() throws IOException {
TestConstants.createXML("catalog.xml");
File xml = new File(TestConstants.XML_FILE);
String md5 = metaFileUtil.generateMD5(xml);
@@ -71,7 +73,7 @@ public class MetaFileUtilTest {
* Test the creation of metadata file.
*/
@Test
- public void step2_testCreateMetaDataFile() throws IOException, JAXBException {
+ public void step2_testCreateMetaDataFile() {
ConcurrentHashMap<String, XmlMetadata> initialMap = TestConstants.getInitialMap();
metaFileUtil.updateMetadataMap(initialMap, "");
metaFileUtil.writeMetadataToFile();
@@ -82,8 +84,8 @@ public class MetaFileUtilTest {
* Validate the content of the file.
*/
@Test
- public void step3_testValidateMetadataFile() throws IOException, ClassNotFoundException, JAXBException {
- ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata("");
+ public void step3_testValidateMetadataFile() {
+ ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata();
Set<String> from = fromFile.keySet();
Set<String> initial = initialMap.keySet();
@@ -100,7 +102,7 @@ public class MetaFileUtilTest {
* Change the xml file and test whether the changes are detected.
*/
@Test
- public void step4_testDetectFileChanges() throws IOException, NoSuchAlgorithmException {
+ public void step4_testDetectFileChanges() throws IOException {
TestConstants.createXML("catalog_edited.xml");
File xml = new File(TestConstants.XML_FILE);
Assert.assertTrue(metaFileUtil.generateMD5(xml).equals(TestConstants.CHANGED_MD5));
@@ -112,7 +114,7 @@ public class MetaFileUtilTest {
@Test
public void step5_testUpdateMetadata()
throws IOException, ClassNotFoundException, NoSuchAlgorithmException, JAXBException {
- ConcurrentHashMap<String, XmlMetadata> fromFileMap = metaFileUtil.getMetadata("");
+ ConcurrentHashMap<String, XmlMetadata> fromFileMap = metaFileUtil.getMetadata();
XmlMetadata modified = fromFileMap.get(TestConstants.XML_FILE);
File xml = new File(TestConstants.XML_FILE);
@@ -122,7 +124,7 @@ public class MetaFileUtilTest {
metaFileUtil.updateMetadataMap(fromFileMap, TestConstants.INDEX_DIR);
- Assert.assertNotNull(metaFileUtil.getMetadata(TestConstants.INDEX_DIR));
+ Assert.assertNotNull(metaFileUtil.getMetadata());
}
@@ -130,8 +132,8 @@ public class MetaFileUtilTest {
* Validate the updated metadata.
*/
@Test
- public void step6_testVerifyMetadataChange() throws IOException, ClassNotFoundException, JAXBException {
- ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata(TestConstants.INDEX_DIR);
+ public void step6_testVerifyMetadataChange() {
+ ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata();
Set<String> from = fromFile.keySet();
Set<String> modified = modifiedMap.keySet();
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
index 68d40b5..b79107e 100644
--- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
+++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
@@ -16,7 +16,13 @@ package org.apache.vxquery.indexing;
import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.concurrent.ConcurrentHashMap;
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/conf/cluster_example.xml
----------------------------------------------------------------------
diff --git a/vxquery-server/src/main/resources/conf/cluster_example.xml b/vxquery-server/src/main/resources/conf/cluster_example.xml
index 18d9173..f43b22a 100644
--- a/vxquery-server/src/main/resources/conf/cluster_example.xml
+++ b/vxquery-server/src/main/resources/conf/cluster_example.xml
@@ -17,6 +17,7 @@
<cluster xmlns="cluster">
<name>local</name>
<username>joe</username>
+ <index_directory>/tmp/indexFolder</index_directory>
<master_node>
<id>master</id>
<client_ip>128.195.52.177</client_ip>
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/conf/local.xml
----------------------------------------------------------------------
diff --git a/vxquery-server/src/main/resources/conf/local.xml b/vxquery-server/src/main/resources/conf/local.xml
index 4a48620..e0d07f8 100644
--- a/vxquery-server/src/main/resources/conf/local.xml
+++ b/vxquery-server/src/main/resources/conf/local.xml
@@ -16,6 +16,7 @@
-->
<cluster xmlns="cluster">
<name>local</name>
+ <index_directory>/tmp/indexFolder</index_directory>
<master_node>
<id>master</id>
<client_ip>127.0.0.1</client_ip>
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/scripts/cluster_actions.py
----------------------------------------------------------------------
diff --git a/vxquery-server/src/main/resources/scripts/cluster_actions.py b/vxquery-server/src/main/resources/scripts/cluster_actions.py
index deeee33..8ce1607 100644
--- a/vxquery-server/src/main/resources/scripts/cluster_actions.py
+++ b/vxquery-server/src/main/resources/scripts/cluster_actions.py
@@ -120,7 +120,7 @@ class ClusterActions:
def start_nc(self, machine, cc):
print "Start Node Controller."
print " " + machine.get_id() + " " + machine.get_ip()
- command = "./vxquery-server/target/appassembler/bin/startnc.sh " + machine.get_id() + " " + machine.get_ip() + " " + cc.get_client_ip() + " \"" + cc.get_client_port() + "\" \"" + machine.get_java_opts() + "\""
+ command = "./vxquery-server/target/appassembler/bin/startnc.sh " + machine.get_id() + " " + machine.get_ip() + " " + cc.get_client_ip() + " \"" + cc.get_client_port() + "\" \"" + machine.get_java_opts() + "\" \"" + self.ci.get_index_directory() + "\""
self.run_remote_command(machine.get_username(), machine.get_id(), command)
def stop_cc_and_all_ncs(self, machine):
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/scripts/cluster_information.py
----------------------------------------------------------------------
diff --git a/vxquery-server/src/main/resources/scripts/cluster_information.py b/vxquery-server/src/main/resources/scripts/cluster_information.py
index 94b231d..d70828d 100644
--- a/vxquery-server/src/main/resources/scripts/cluster_information.py
+++ b/vxquery-server/src/main/resources/scripts/cluster_information.py
@@ -28,6 +28,9 @@ class ClusterInformation:
def get_java_opts(self):
return get_tag_text(self.config, "java_opts")
+ def get_index_directory(self):
+ return get_tag_text(self.config, "index_directory");
+
def get_master_node_machine(self):
master_node = self.config.getElementsByTagName("master_node")[0]
id = NodeXmlReader.get_cluster_id(master_node)
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/scripts/startnc.sh
----------------------------------------------------------------------
diff --git a/vxquery-server/src/main/resources/scripts/startnc.sh b/vxquery-server/src/main/resources/scripts/startnc.sh
index a199ec8..d719d22 100755
--- a/vxquery-server/src/main/resources/scripts/startnc.sh
+++ b/vxquery-server/src/main/resources/scripts/startnc.sh
@@ -24,6 +24,7 @@ IPADDR=$2
CCHOST=$3
CCPORT=$4
J_OPTS=$5
+IO_DEVICES=$6
#Import cluster properties
MYDIR="$(dirname -- $0)"
@@ -62,6 +63,9 @@ NC_OPTIONS=" -cc-host ${CCHOST} -cluster-net-ip-address ${IPADDR} -data-ip-addr
NC_OPTIONS=" ${NC_OPTIONS} ${NCOPTS} "
+[ "${IO_DEVICES}" ] &&
+ NC_OPTIONS=" ${NC_OPTIONS} -iodevices ${IO_DEVICES} "
+
echo "${JAVA_OPTS}" &> ${NCLOGS_DIR}/nc.log
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
index 75dce1b..d7db8a8 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
@@ -17,16 +17,16 @@
package org.apache.vxquery.xtest;
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.NodeControllerService;
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
public class TestClusterUtil {
private static int clientNetPort = 39000;
@@ -34,6 +34,7 @@ public class TestClusterUtil {
private static int profileDumpPeriod = 10000;
private static String ccHost = "localhost";
private static String nodeId = "nc1";
+ private static String ioDevices = "target/tmp/indexFolder";
private TestClusterUtil() {
}
@@ -58,6 +59,7 @@ public class TestClusterUtil {
ncConfig1.dataIPAddress = publicAddress;
ncConfig1.resultIPAddress = publicAddress;
ncConfig1.nodeId = nodeId;
+ ncConfig1.ioDevices = ioDevices;
return ncConfig1;
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
index c6b339b..78ed8ff 100644
--- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
+++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
@@ -80,6 +80,7 @@ public abstract class AbstractXQueryTest {
case EXPECTED_RESULT_GOT_SAME_RESULT:
break;
case NO_RESULT_FILE:
+ fail(result.state + " (" + result.time + " ms): " + result.testCase.getXQueryDisplayName());
break;
}
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt
new file mode 100644
index 0000000..f32a580
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt
@@ -0,0 +1 @@
+true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt
new file mode 100644
index 0000000..f32a580
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt
@@ -0,0 +1 @@
+true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt
new file mode 100644
index 0000000..17d6fec
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt
@@ -0,0 +1 @@
+src/test/resources/TestSources/ghcnd
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt
@@ -0,0 +1 @@
+
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt
new file mode 100644
index 0000000..f32a580
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt
@@ -0,0 +1 @@
+true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt
new file mode 100644
index 0000000..baf9dca
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt
new file mode 100644
index 0000000..ef8dde4
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt
@@ -0,0 +1 @@
+<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt
@@ -0,0 +1 @@
+2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt
new file mode 100644
index 0000000..f30101c
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt
@@ -0,0 +1 @@
+3.3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt
new file mode 100644
index 0000000..c84c360
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt
new file mode 100644
index 0000000..9abedff
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt
@@ -0,0 +1,2 @@
+<station><id>GHCND:US000000001</id><displayName>Station 1</displayName><latitude>10.000</latitude><longitude>-10.000</longitude><elevation>1000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 1</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
+<station><id>GHCND:US000000002</id><displayName>Station 2</displayName><latitude>20.000</latitude><longitude>-20.000</longitude><elevation>2000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 2</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt
new file mode 100644
index 0000000..c84c360
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt
new file mode 100644
index 0000000..0be5d98
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt
@@ -0,0 +1,2 @@
+true
+true
\ No newline at end of file