You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by ch...@apache.org on 2017/07/11 17:24:34 UTC
[1/3] vxquery git commit: [VXQUERY-196][VXQUERY-204][VXQUERY-228]
Cleaning up indexing query statements
Repository: vxquery
Updated Branches:
refs/heads/master 37642bee2 -> 4a38f670c
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
index 6558274..1d66c4e 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
@@ -16,7 +16,6 @@
*/
package org.apache.vxquery.runtime.functions.util;
-import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.File;
import java.io.FileInputStream;
@@ -37,7 +36,7 @@ import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.data.std.util.GrowableArray;
import org.apache.hyracks.data.std.util.UTF8StringBuilder;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+import org.apache.hyracks.util.string.UTF8StringUtil;
import org.apache.vxquery.context.DynamicContext;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
import org.apache.vxquery.datamodel.accessors.TypedPointables;
@@ -485,6 +484,15 @@ public class FunctionHelper {
return true;
}
+ public static String getStringFromBytes(byte[] bytes) {
+ if (bytes == null) {
+ return null;
+ }
+ StringBuilder sb = new StringBuilder();
+ UTF8StringUtil.toString(sb, bytes, 0);
+ return sb.toString();
+ }
+
public static boolean compareTaggedValues(AbstractValueComparisonOperation aOp, TaggedValuePointable tvp1,
TaggedValuePointable tvp2, DynamicContext dCtx, TypedPointables tp1, TypedPointables tp2)
throws SystemException {
@@ -1215,13 +1223,12 @@ public class FunctionHelper {
System.err.println(" printUTF8String END");
}
- public static void readInDocFromPointable(UTF8StringPointable stringp, ByteBufferInputStream bbis,
- DataInputStream di, ArrayBackedValueStorage abvs, IParser parser) throws IOException {
- readInDocFromString(stringp.toString(), bbis, di, abvs, parser);
+ public static void readInDocFromPointable(UTF8StringPointable stringp, ArrayBackedValueStorage abvs,
+ IParser parser) throws IOException {
+ readInDocFromString(stringp.toString(), abvs, parser);
}
- public static void readInDocFromString(String fName, ByteBufferInputStream bbis, DataInputStream di,
- ArrayBackedValueStorage abvs, IParser parser) throws IOException {
+ public static void readInDocFromString(String fName, ArrayBackedValueStorage abvs, IParser parser) throws IOException {
Reader input;
if (!fName.contains("hdfs:/")) {
File file = new File(fName);
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
index 84c8ddf..9e21f53 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
@@ -129,8 +129,6 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
public SAXContentHandler(boolean attachTypes, ITreeNodeIdProvider nodeIdProvider, IFrameFieldAppender appender,
List<SequenceType> childSequenceTypes) {
this(attachTypes, nodeIdProvider, false);
-
- // Frame writing variables
this.appender = appender;
setChildPathSteps(childSequenceTypes);
}
@@ -297,10 +295,11 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
/**
* The filter settings here are similar to one in the class linked below.
- *
+ *
+ * @throws SAXException
* @see org.apache.vxquery.runtime.functions.step.NodeTestFilter.java
*/
- private boolean startElementChildPathStep(String uri, String localName) {
+ private boolean startElementChildPathStep(String uri, String localName) throws SAXException {
if (subElement != null && depth <= subElement.length) {
// Check path step if it exists.
subElement[depth - 1] = true;
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
index 0c252b4..d3d02ae 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
@@ -222,7 +222,7 @@ public class XMLQueryCompiler {
}
}
- public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel)
+ public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel, List<String> collections)
throws AlgebricksException, SystemException {
moduleNode = XMLQueryParser.parse(name, query);
listener.notifyParseResult(moduleNode);
@@ -230,7 +230,7 @@ public class XMLQueryCompiler {
pprinter = new LogicalOperatorPrettyPrintVisitor(new AlgebricksAppendable(),
new VXQueryLogicalExpressionPrettyPrintVisitor(module.getModuleContext()));
VXQueryMetadataProvider mdProvider = new VXQueryMetadataProvider(nodeList, ccb.getSourceFileMap(),
- module.getModuleContext(), this.hdfsConf, nodeControllerInfos);
+ module.getModuleContext(), this.hdfsConf, nodeControllerInfos, collections);
compiler = cFactory.createCompiler(module.getBody(), mdProvider, 0);
listener.notifyTranslationResult(module);
XMLQueryTypeChecker.typeCheckModule(module);
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
index 60f39f8..45d553f 100644
--- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
+++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
@@ -23,8 +23,8 @@ import java.util.concurrent.ConcurrentHashMap;
import javax.xml.bind.JAXBException;
import org.apache.commons.io.FileUtils;
-import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil;
-import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
+import org.apache.vxquery.runtime.functions.index.update.MetaFileUtil;
+import org.apache.vxquery.runtime.functions.index.update.XmlMetadata;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
index b79107e..1b3e0b7 100644
--- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
+++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
@@ -14,8 +14,6 @@
*/
package org.apache.vxquery.indexing;
-import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
-
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
@@ -27,6 +25,8 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.vxquery.runtime.functions.index.update.XmlMetadata;
+
/**
* TestConstants and methods which will be used in indexing test cases.
*/
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java b/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java
index 7646f97..1ee35a4 100644
--- a/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java
+++ b/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java
@@ -142,6 +142,6 @@ public class SimpleXQueryTest {
XMLQueryCompiler compiler = new XMLQueryCompiler(null, nodeControllerInfos, 65536);
CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
new ResultSetId(System.nanoTime()), null);
- compiler.compile(testName, new StringReader(query), ccb, Integer.MAX_VALUE);
+ compiler.compile(testName, new StringReader(query), ccb, Integer.MAX_VALUE, null);
}
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
index e4ba6eb..fa0a900 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
@@ -14,16 +14,22 @@
*/
package org.apache.vxquery.xtest;
+import java.io.File;
import java.io.FileInputStream;
+import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.EnumSet;
+import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.client.NodeControllerInfo;
import org.apache.hyracks.api.comm.IFrame;
@@ -53,13 +59,14 @@ import org.apache.vxquery.xmlquery.query.XMLQueryCompiler;
public class TestRunner {
private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})");
-
+ private List<String> collectionList;
private XTestOptions opts;
private IHyracksClientConnection hcc;
private IHyracksDataset hds;
public TestRunner(XTestOptions opts) throws UnknownHostException {
this.opts = opts;
+ this.collectionList = new ArrayList<String>();
}
public void open() throws Exception {
@@ -67,8 +74,46 @@ public class TestRunner {
hds = TestClusterUtil.getDataset();
}
+ protected static TestConfiguration getIndexConfiguration(TestCase testCase) {
+ XTestOptions opts = new XTestOptions();
+ opts.verbose = false;
+ opts.threads = 1;
+ opts.showQuery = true;
+ opts.showResult = true;
+ opts.hdfsConf = "src/test/resources/hadoop/conf";
+ opts.catalog = StringUtils.join(new String[] { "src", "test", "resources", "VXQueryCatalog.xml" },
+ File.separator);
+ TestConfiguration indexConf = new TestConfiguration();
+ indexConf.options = opts;
+ String baseDir = new File(opts.catalog).getParent();
+ try {
+ String root = new File(baseDir).getCanonicalPath();
+ indexConf.testRoot = new File(root + "/./");
+ indexConf.resultOffsetPath = new File(root + "/./ExpectedResults/");
+ indexConf.sourceFileMap = testCase.getSourceFileMap();
+ indexConf.xqueryFileExtension = ".xq";
+ indexConf.xqueryxFileExtension = "xqx";
+ indexConf.xqueryQueryOffsetPath = new File(root + "/./Queries/XQuery/");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return indexConf;
+
+ }
+
public TestCaseResult run(final TestCase testCase) {
TestCaseResult res = new TestCaseResult(testCase);
+ TestCase testCaseIndex = new TestCase(getIndexConfiguration(testCase));
+ testCaseIndex.setFolder("Indexing/Partition-1/");
+ testCaseIndex.setName("showIndexes");
+ runQuery(testCaseIndex, res);
+ String[] collections = res.result.split("\n");
+ this.collectionList = Arrays.asList(collections);
+ runQueries(testCase, res);
+ return res;
+ }
+
+ public void runQuery(TestCase testCase, TestCaseResult res) {
if (opts.verbose) {
System.err.println("Starting " + testCase.getXQueryDisplayName());
}
@@ -78,6 +123,7 @@ public class TestRunner {
try {
try {
if (opts.showQuery) {
+
FileInputStream query = new FileInputStream(testCase.getXQueryFile());
System.err.println("***Query for " + testCase.getXQueryDisplayName() + ": ");
System.err.println(IOUtils.toString(query, "UTF-8"));
@@ -98,7 +144,7 @@ public class TestRunner {
CompilerControlBlock ccb = new CompilerControlBlock(
new StaticContextImpl(RootStaticContextImpl.INSTANCE),
new ResultSetId(testCase.getXQueryDisplayName().hashCode()), testCase.getSourceFileMap());
- compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel);
+ compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel, collectionList);
JobSpecification spec = compiler.getModule().getHyracksJobSpecification();
in.close();
@@ -172,7 +218,11 @@ public class TestRunner {
System.err.println(res.result);
}
}
- return res;
+
+ }
+
+ public void runQueries(TestCase testCase, TestCaseResult res) {
+ runQuery(testCase, res);
}
public void close() throws Exception {
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt
new file mode 100644
index 0000000..baf9dca
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt
new file mode 100644
index 0000000..baf9dca
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt
new file mode 100644
index 0000000..baf9dca
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
index 63fdda7..96a7671 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
@@ -16,10 +16,10 @@
under the License. :)
(: Search Lucene Index :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
let $datetime := xs:dateTime(fn:data($r/date))
where $r/station eq "GHCND:AS000000003"
and fn:year-from-dateTime($datetime) ge 2000
and fn:month-from-dateTime($datetime) eq 3
and fn:day-from-dateTime($datetime) eq 3
-return $r
\ No newline at end of file
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq
new file mode 100644
index 0000000..7a2bb2c
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd")/dataCollection/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003"
+ and fn:year-from-dateTime($datetime) ge 2000
+ and fn:month-from-dateTime($datetime) eq 3
+ and fn:day-from-dateTime($datetime) eq 3
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
index cf41536..464c1cc 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
@@ -19,6 +19,6 @@
(: Find all reading for hurricane force wind warning or extreme wind warning. :)
(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :)
(: meters per second). (Wind value is in tenth of a meter per second) :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
index 5c99d9a..eeb019b 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
@@ -19,7 +19,7 @@
(: Find the annual precipitation (PRCP) for a Seattle using the airport :)
(: station (US000000002) for 2002. :)
fn:sum(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
where $r/station eq "GHCND:US000000002"
and $r/dataType eq "PRCP"
and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
index 39e5d17..06284ae 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
@@ -18,7 +18,7 @@
(: Search Lucene Index :)
(: Find the highest recorded temperature (TMAX) in Celsius. :)
fn:max(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
where $r/dataType eq "TMAX"
return $r/value
) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
index 63aeca5..3a4ae05 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
@@ -18,6 +18,6 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
index c81f271..9090edd 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
@@ -18,7 +18,7 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station
+for $s in collection("src/test/resources/TestSources/ghcnd")/stationCollection/station
where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
order by $s/id
return $s
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
index dd6b5f9..7c703b6 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
@@ -18,8 +18,8 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $s in collection("src/test/resources/TestSources/ghcnd")/stationCollection/station
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
where $s/id eq $r/station
and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
index fecb56d..4718240 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
@@ -16,7 +16,7 @@
under the License. :)
(: Search Lucene Index :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
let $datetime := xs:dateTime(fn:data($r/date))
where $r/station eq "GHCND:AS000000003"
and fn:year-from-dateTime($datetime) ge 2000
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq
new file mode 100644
index 0000000..0e42155
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003"
+ and fn:year-from-dateTime($datetime) ge 2000
+ and fn:month-from-dateTime($datetime) eq 3
+ and fn:day-from-dateTime($datetime) eq 3
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
index 75c7a64..37e5626 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
@@ -19,6 +19,6 @@
(: Find all reading for hurricane force wind warning or extreme wind warning. :)
(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :)
(: meters per second). (Wind value is in tenth of a meter per second) :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
index 28f7473..358d3f3 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
@@ -19,7 +19,7 @@
(: Find the annual precipitation (PRCP) for a Seattle using the airport :)
(: station (US000000002) for 2002. :)
fn:sum(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
where $r/station eq "GHCND:US000000002"
and $r/dataType eq "PRCP"
and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
index 317a141..bd5ba1c 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
@@ -18,7 +18,7 @@
(: Search Lucene Index :)
(: Find the highest recorded temperature (TMAX) in Celsius. :)
fn:max(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
where $r/dataType eq "TMAX"
return $r/value
) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
index 2deb4c3..77f6c2f 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
@@ -18,6 +18,6 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
index a0ce1e9..c1a45f4 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
@@ -18,6 +18,6 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/stationCollection/station
where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
return $s
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
index b3e622c..a776ab9 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
@@ -18,8 +18,8 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/stationCollection/station
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
where $s/id eq $r/station
and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
index 0cccbc5..2bc9ce7 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
@@ -16,7 +16,7 @@
under the License. :)
(: Search Lucene Index :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
let $datetime := xs:dateTime(fn:data($r/date))
where $r/station eq "GHCND:AS000000003"
and fn:year-from-dateTime($datetime) ge 2000
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq
new file mode 100644
index 0000000..e740365
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003"
+ and fn:year-from-dateTime($datetime) ge 2000
+ and fn:month-from-dateTime($datetime) eq 3
+ and fn:day-from-dateTime($datetime) eq 3
+return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
index c282e31..a1b86ac 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
@@ -19,6 +19,6 @@
(: Find all reading for hurricane force wind warning or extreme wind warning. :)
(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :)
(: meters per second). (Wind value is in tenth of a meter per second) :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
index 33ea1c9..9cc2b8e 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
@@ -19,7 +19,7 @@
(: Find the annual precipitation (PRCP) for a Seattle using the airport :)
(: station (US000000002) for 2002. :)
fn:sum(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
where $r/station eq "GHCND:US000000002"
and $r/dataType eq "PRCP"
and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
index d213082..dd26e87 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
@@ -18,7 +18,7 @@
(: Search Lucene Index :)
(: Find the highest recorded temperature (TMAX) in Celsius. :)
fn:max(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
where $r/dataType eq "TMAX"
return $r/value
) div 10
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
index 1d98682..4a9d224 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
@@ -18,6 +18,6 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
return $r
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
index abe2184..4407079 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
@@ -18,6 +18,6 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/stationCollection/station
where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
return $s
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
index 7b40ca0..664b150 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
@@ -18,8 +18,8 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/stationCollection/station
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
where $s/id eq $r/station
and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/VXQueryCatalog.xml
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml
index 5ecdb94..8e28135 100644
--- a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml
+++ b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml
@@ -353,7 +353,7 @@
&LibrariesInJSONiq;
</test-group>
</test-group>
- <test-group name="SerializationQueries" featureOwner="Christina Pavlopoulou">
+ <test-group name="SerializationQueries" featureOwner="Christina Pavlopoulou">
<GroupInfo>
<title>Serialize Function Queries</title>
<description/>
@@ -365,7 +365,7 @@
</GroupInfo>
&SerializationQueries;
</test-group>
- </test-group>
+ </test-group>
<test-group name="XMLInJSONQueries" featureOwner="Riyafa Abdul Hameed">
<GroupInfo>
<title>XML in JSON</title>
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml b/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
index 7cf6bf6..cc6b65b 100644
--- a/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
+++ b/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
@@ -35,6 +35,11 @@
<query name="useIndex1" date="2016-05-26"/>
<output-file compare="Text">useIndex1.txt</output-file>
</test-case>
+ <test-case name="use-index-1-user" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex1_user" date="2016-05-26"/>
+ <output-file compare="Text">useIndex1_user.txt</output-file>
+ </test-case>
<test-case name="use-index-2" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex2" date="2016-05-26"/>
@@ -95,6 +100,11 @@
<query name="useIndex1" date="2016-05-26"/>
<output-file compare="Text">useIndex1.txt</output-file>
</test-case>
+ <test-case name="use-index-1-user" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex1_user" date="2016-05-26"/>
+ <output-file compare="Text">useIndex1_user.txt</output-file>
+ </test-case>
<test-case name="use-index-2" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex2" date="2016-05-26"/>
@@ -145,6 +155,11 @@
<query name="useIndex1" date="2016-05-26"/>
<output-file compare="Text">useIndex1.txt</output-file>
</test-case>
+ <test-case name="use-index-1-user" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex1_user" date="2016-05-26"/>
+ <output-file compare="Text">useIndex1_user.txt</output-file>
+ </test-case>
<test-case name="use-index-2" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex2" date="2016-05-26"/>
[2/3] vxquery git commit: [VXQUERY-196][VXQUERY-204][VXQUERY-228]
Cleaning up indexing query statements
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java
new file mode 100644
index 0000000..1d58433
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.centralizer;
+
+import java.io.DataOutput;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.vxquery.datamodel.builders.atomic.StringValueBuilder;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.datamodel.values.ValueTag;
+
+/**
+ * Class for maintaining the centralized index information file.
+ * Index centralization procedure.
+ * User can specify the collection directory in VXQuery.java, ncConfig.ioDevices = < index_directory > .
+ * Then all the indexes will be created in that particular directory in sub-folders corresponding to collections.
+ * There will be a single xml file, located in the directory specified in local.xml, which contains all information
+ * about the existing indexes.
+ * This class can be used to read, add, delete, modify the entries and write the file back to the disk.
+ */
+public class IndexCentralizerUtil {
+
+ private static final String FILE_NAME = "VXQuery-Index-Directory.xml";
+ private final List<String> collections = new ArrayList<>();
+ private static final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil");
+ private File xmlFile;
+ private String indexPath;
+ public static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>();
+ private static final StringValueBuilder svb = new StringValueBuilder();
+ private final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ private final DataOutput output = abvs.getDataOutput();
+
+ public IndexCentralizerUtil(File index) {
+ indexPath = index.getPath();
+ if (!index.exists()) {
+ try {
+ FileUtils.forceMkdir(index);
+ } catch (IOException e) {
+ LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + indexPath + " " + e);
+ }
+ }
+ xmlFile = new File(index.getPath() + "/" + FILE_NAME);
+ }
+
+ /**
+ * Get the index directory containing index of the given collection
+ *
+ * @param collection
+ * : Collection folder
+ * @return Index folder.
+ */
+ public String getIndexForCollection(String collection) {
+ if (indexCollectionMap.size() > 0 && indexCollectionMap.containsKey(collection)) {
+ return indexCollectionMap.get(collection).getIndex();
+ }
+ return null;
+ }
+
+ /**
+ * Put the index location corresponding to given collection.
+ * Index location is created by using the last 100 characters of collection.
+ *
+ * @param collection
+ * : Collection directory
+ * @return index
+ */
+ public String putIndexForCollection(String collection) {
+ int length = collection.replaceAll("/", "").length();
+ String index = collection.replaceAll("/", "");
+ index = indexPath + "/" + (length > 100 ? index.substring(length - 100) : index);
+ IndexLocator il = new IndexLocator();
+ il.setCollection(collection);
+ il.setIndex(index);
+ if (indexCollectionMap.get(collection) != null) {
+ return index;
+ }
+ indexCollectionMap.put(collection, il);
+ return index;
+ }
+
+ /**
+ * Remove the entry for given collection directory.
+ *
+ * @param collection
+ * : Collection directory
+ */
+ public void deleteEntryForCollection(String collection) {
+ indexCollectionMap.remove(collection);
+ }
+
+ /**
+ * Prints all collections which have an index created.
+ *
+ * @param sb
+ * : The output is stored in a sequence
+ * @throws IOException
+ * : If writing the dataOutput generates {@link IOException}
+ */
+ public void getAllCollections(SequenceBuilder sb) throws IOException {
+ for (String s : collections) {
+ abvs.reset();
+ output.write(ValueTag.XS_STRING_TAG);
+ svb.write(s, output);
+ sb.addItem(abvs);
+ }
+ }
+
+ /**
+ * Read the collection, index directory file and populate the HashMap.
+ */
+ public void readIndexDirectory() {
+ if (xmlFile.exists()) {
+ try {
+ JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class);
+ Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
+ IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(xmlFile);
+
+ for (IndexLocator il : indexDirectory.getDirectory()) {
+ indexCollectionMap.put(il.getCollection(), il);
+ this.collections.add(il.getCollection());
+ }
+ } catch (JAXBException e) {
+ LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e);
+ }
+ }
+
+ }
+
+ /**
+ * Write back the contents of the HashMap to the file.
+ */
+ public void writeIndexDirectory() {
+ IndexDirectory id = new IndexDirectory();
+ List<IndexLocator> indexLocators = new ArrayList<>(indexCollectionMap.values());
+ id.setDirectory(indexLocators);
+ try {
+ FileOutputStream fileOutputStream = new FileOutputStream(this.xmlFile);
+ JAXBContext context = JAXBContext.newInstance(IndexDirectory.class);
+ Marshaller jaxbMarshaller = context.createMarshaller();
+ jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+ jaxbMarshaller.marshal(id, fileOutputStream);
+ } catch (JAXBException | FileNotFoundException e) {
+ LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java
new file mode 100644
index 0000000..d118926
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.centralizer;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "indexes")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class IndexDirectory implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @XmlElement(name = "index", type = IndexLocator.class)
+ private List<IndexLocator> directory = new ArrayList<>();
+
+ public List<IndexLocator> getDirectory() {
+ return directory;
+ }
+
+ public void setDirectory(List<IndexLocator> directory) {
+ this.directory = directory;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java
new file mode 100644
index 0000000..49dbecd
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.centralizer;
+
+import java.io.Serializable;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlRootElement(name = "Entry")
+public class IndexLocator implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @XmlAttribute
+ private String collection;
+
+ @XmlAttribute
+ private String index;
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public void setCollection(String collection) {
+ this.collection = collection;
+ }
+
+ public String getIndex() {
+ return index;
+ }
+
+ public void setIndex(String index) {
+ this.index = index;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
deleted file mode 100644
index 51510d5..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.vxquery.runtime.functions.index.indexCentralizer;
-
-import java.io.DataOutput;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
-import javax.xml.bind.Unmarshaller;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.vxquery.datamodel.builders.atomic.StringValueBuilder;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.ValueTag;
-
-/**
- * Class for maintaining the centralized index information file.
- * Index centralization procedure.
- * User can specify the collection directory in VXQuery.java, ncConfig.ioDevices = < index_directory > .
- * Then all the indexes will be created in that particular directory in sub-folders corresponding to collections.
- * There will be a single xml file, located in the directory specified in local.xml, which contains all information
- * about the existing indexes.
- * This class can be used to read, add, delete, modify the entries and write the file back to the disk.
- */
-public class IndexCentralizerUtil {
-
- private final String FILE_NAME = "VXQuery-Index-Directory.xml";
- private final List<String> collections = new ArrayList<>();
- private final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil");
- private File XML_FILE;
- private String INDEX_LOCATION;
- private static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>();
-
- public IndexCentralizerUtil(File index) {
- this.INDEX_LOCATION = index.getPath();
- if (!index.exists()) {
- try {
- FileUtils.forceMkdir(index);
- } catch (IOException e) {
- LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + INDEX_LOCATION + " " + e);
- }
- }
- XML_FILE = new File(index.getPath() + "/" + FILE_NAME);
- }
-
- /**
- * Get the index directory containing index of the given collection
- *
- * @param collection : Collection folder
- * @return Index folder.
- */
- public String getIndexForCollection(String collection) {
- return indexCollectionMap.get(collection).getIndex();
- }
-
- /**
- * Put the index location corresponding to given collection.
- * Index location is created by using the last 100 characters of collection.
- *
- * @param collection : Collection directory
- * @return index
- */
- public String putIndexForCollection(String collection) {
- int length = collection.replaceAll("/", "").length();
- String index = collection.replaceAll("/", "");
- index = INDEX_LOCATION + "/" + (length > 100 ? index.substring(length - 100) : index);
- IndexLocator il = new IndexLocator();
- il.setCollection(collection);
- il.setIndex(index);
- if (indexCollectionMap.get(collection) != null) {
- return index;
- }
- indexCollectionMap.put(collection, il);
- return index;
- }
-
- /**
- * Remove the entry for given collection directory.
- *
- * @param collection : Collection directory
- */
- public void deleteEntryForCollection(String collection) {
- indexCollectionMap.remove(collection);
- }
-
- /**
- * Prints all collections which have an index created.
- * @param sb : The output is stored in a sequence
- * @throws IOException : If writing the dataOutput generates {@link IOException}
- */
- public void getAllCollections(SequenceBuilder sb) throws IOException {
- for (String s : collections) {
- StringValueBuilder svb = new StringValueBuilder();
- ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
- DataOutput output = abvs.getDataOutput();
- output.write(ValueTag.XS_STRING_TAG);
- svb.write(s, output);
- sb.addItem(abvs);
- }
- }
-
- /**
- * Read the collection, index directory file and populate the HashMap.
- */
- public void readIndexDirectory() {
- if (this.XML_FILE.exists()) {
- try {
- JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class);
- Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
- IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(this.XML_FILE);
-
- for (IndexLocator il : indexDirectory.getDirectory()) {
- indexCollectionMap.put(il.getCollection(), il);
- this.collections.add(il.getCollection());
- }
- } catch (JAXBException e) {
- LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e);
- }
- }
-
- }
-
- /**
- * Write back the contents of the HashMap to the file.
- */
- public void writeIndexDirectory() {
- IndexDirectory id = new IndexDirectory();
- List<IndexLocator> indexLocators = new ArrayList<>(indexCollectionMap.values());
- id.setDirectory(indexLocators);
- try {
- FileOutputStream fileOutputStream = new FileOutputStream(this.XML_FILE);
- JAXBContext context = JAXBContext.newInstance(IndexDirectory.class);
- Marshaller jaxbMarshaller = context.createMarshaller();
- jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
- jaxbMarshaller.marshal(id, fileOutputStream);
- } catch (JAXBException | FileNotFoundException e) {
- LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
deleted file mode 100644
index 54d9ad9..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.vxquery.runtime.functions.index.indexCentralizer;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-@XmlRootElement(name = "indexes")
-@XmlAccessorType(XmlAccessType.FIELD)
-public class IndexDirectory implements Serializable{
-
- @XmlElement(name = "index", type = IndexLocator.class)
- private List<IndexLocator> directory = new ArrayList<>();
-
- public List<IndexLocator> getDirectory() {
- return directory;
- }
-
-
- public void setDirectory(List<IndexLocator> directory) {
- this.directory = directory;
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
deleted file mode 100644
index 1a33c8b..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.vxquery.runtime.functions.index.indexCentralizer;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlAttribute;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
-
-@XmlAccessorType(XmlAccessType.FIELD)
-@XmlRootElement(name = "Entry")
-public class IndexLocator implements Serializable{
-
- @XmlAttribute
- private String collection;
-
- @XmlAttribute
- private String index;
-
- public String getCollection() {
- return collection;
- }
-
- public void setCollection(String collection) {
- this.collection = collection;
- }
-
- public String getIndex() {
- return index;
- }
-
- public void setIndex(String index) {
- this.index = index;
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java
new file mode 100644
index 0000000..0346a62
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java
@@ -0,0 +1,28 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index.update;
+
+/**
+ * Constants used in updating index
+ */
+public class Constants {
+ public static final String FIELD_PATH = "path";
+ public static final String META_FILE_NAME = "vxquery_index.xml";
+
+ private Constants() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java
new file mode 100644
index 0000000..65a8325
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java
@@ -0,0 +1,304 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index.update;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.index.IndexDocumentBuilder;
+import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer;
+import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
+import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
+
+/**
+ * Update the index if the source files are changed.
+ */
+public class IndexUpdater {
+ private MetaFileUtil metaFileUtil;
+ private ConcurrentHashMap<String, XmlMetadata> metadataMap;
+ private IPointable result;
+ private final SequenceBuilder sb = new SequenceBuilder();
+ private ArrayBackedValueStorage abvs;
+ private ITreeNodeIdProvider nodeIdProvider;
+ private ArrayBackedValueStorage abvsFileNode;
+ private String nodeId;
+ private IndexWriter indexWriter;
+ private Set<String> pathsFromFileList;
+ private String collectionFolder;
+ private String indexFolder;
+ private final Logger LOGGER = Logger.getLogger("Index Updater");
+ private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
+ private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
+
+ public IndexUpdater(String indexFolder, IPointable result, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider,
+ ArrayBackedValueStorage abvsFileNode, String nodeId) {
+ this.indexFolder = indexFolder;
+ this.result = result;
+ this.abvs = abvs;
+ this.nodeIdProvider = nodeIdProvider;
+ this.abvsFileNode = abvsFileNode;
+ this.nodeId = nodeId;
+ this.pathsFromFileList = new HashSet<>();
+ }
+
+ /**
+ * Perform the initial configuration for index update/ delete processes.
+ *
+ * @throws IOException
+ * : If getting the index folder generates {@link IOException}
+ */
+ public void setup() throws IOException {
+ // Read the metadata file and load the metadata map into memory.
+ metaFileUtil = new MetaFileUtil(indexFolder);
+ metaFileUtil.readMetadataFile();
+ metadataMap = metaFileUtil.getMetadata();
+
+ // Retrieve the collection folder path.
+ // Remove the entry for ease of the next steps.
+ collectionFolder = metaFileUtil.getCollection();
+
+ abvs.reset();
+ sb.reset(abvs);
+
+ Directory fsdir = FSDirectory.open(Paths.get(indexFolder));
+ indexWriter = new IndexWriter(fsdir, new IndexWriterConfig(new CaseSensitiveAnalyzer())
+ .setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND));
+ }
+
+ /**
+ * Wrapper for update index function.
+ *
+ * @throws IOException
+ * : If the directory doesn't exist
+ */
+ public void updateIndex() throws IOException {
+ File collectionDirectory = new File(collectionFolder);
+ if (!collectionDirectory.exists()) {
+ throw new IOException("The collection directory (" + collectionFolder + ") does not exist.");
+ }
+
+ //Execute update index process
+ updateIndex(collectionDirectory);
+
+ //Detect deleted files and execute the delete index process.
+ deleteIndexOfDeletedFiles(metadataMap.keySet(), pathsFromFileList);
+ updateMetadataFile();
+ }
+
+ /**
+ * Close opened IndexWriter and terminate the index update/ delete process.
+ *
+ * @throws IOException
+ * : If exiting the index folder generates {@link IOException}
+ */
+ public void exit() throws IOException {
+ indexWriter.forceMerge(1);
+
+ indexWriter.close();
+
+ sb.finish();
+ result.set(abvs);
+ }
+
+ /**
+ * Functional wrapper to update Metadata file.
+ *
+ * @throws IOException
+ * : If updating metadata folder generates {@link IOException}
+ */
+ public synchronized void updateMetadataFile() throws IOException {
+ //Write the updated metadata to the file.
+ metaFileUtil.updateMetadataMap(metadataMap, indexFolder);
+ metaFileUtil.writeMetadataToFile();
+ }
+
+ /**
+ * Check the collection for changes.
+ * If changes are detected, update the index
+ *
+ * @param collection
+ * : Collection folder path
+ */
+ private void updateIndex(File collection) throws IOException {
+
+ File[] list = collection.listFiles();
+
+ assert list != null;
+ for (File file : list) {
+ pathsFromFileList.add(file.getCanonicalPath());
+ if (indexConstructorUtil.readableXmlFile(file.getCanonicalPath())) {
+ XmlMetadata data = metadataMap.get(file.getCanonicalPath());
+ String md5 = metaFileUtil.generateMD5(file);
+
+ abvsFileNode.reset();
+
+ IndexDocumentBuilder indexDocumentBuilder;
+ if (data != null) {
+
+ // This case checks whether the file has been changed.
+ // If the file has changed, delete the existing document, create a new index document and add it
+ // to the current index.
+ // At the same time, update the metadata for the file.
+ if (!md5.equals(data.getMd5())) {
+
+ //Update index corresponding to the xml file.
+ indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath()));
+ indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, abvsFileNode,
+ nodeIdProvider, nodeId);
+ indexDocumentBuilder.printStart();
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.log(Level.DEBUG, "New Index is created for updated file " + file.getCanonicalPath());
+ }
+
+ //Update the metadata map.
+ XmlMetadata metadata = updateEntry(file, data);
+ metadataMap.replace(file.getCanonicalPath(), metadata);
+
+ }
+ } else {
+
+ // In this case, the xml file has not added to the index. (It is a newly added file)
+ // Therefore generate a new index for this file and add it to the existing index.
+ indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, abvsFileNode, nodeIdProvider,
+ nodeId);
+ indexDocumentBuilder.printStart();
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.log(Level.DEBUG, "New Index is created for newly added file " + file.getCanonicalPath());
+ }
+
+ XmlMetadata metadata = updateEntry(file, null);
+ metadataMap.put(file.getCanonicalPath(), metadata);
+ }
+ } else if (file.isDirectory()) {
+ updateIndex(file);
+ }
+ }
+ }
+
+ /**
+ * Update the current XmlMetadata object related to the currently reading XML file.
+ *
+ * @param file
+ * : XML file
+ * @param metadata
+ * : Existing metadata object
+ * @return : XML metadata object with updated fields.
+ * @throws IOException
+ * : If getting the file info generates {@link IOException}
+ */
+ private XmlMetadata updateEntry(File file, XmlMetadata metadataArg) throws IOException {
+ XmlMetadata metadata = metadataArg;
+ if (metadata == null) {
+ metadata = new XmlMetadata();
+ }
+ metadata.setFileName(file.getName());
+ metadata.setPath(file.getCanonicalPath());
+ metadata.setMd5(metaFileUtil.generateMD5(file));
+ metadata.setLastModified(sdf.format(file.lastModified()));
+ return metadata;
+ }
+
+ /**
+ * Delete the index of deleted files.
+ *
+ * @param pathsFromMap
+ * : Set of paths taken from metafile.
+ * @param pathsFromFileList
+ * : Set of paths taken from list of existing files.
+ * @throws IOException
+ * : If deleting Documents generates {@link IOException}
+ */
+ private void deleteIndexOfDeletedFiles(Set<String> pathsFromMap, Set<String> pathsFromFileList) throws IOException {
+ Set<String> sfm = new HashSet<>(pathsFromMap);
+
+ // If any file has been deleted from the collection, the number of files stored in metadata is higher than
+ // the actual number of files.
+ // With set difference, the paths of deleted files are taken from the stored metadata.
+ // Delete the corresponding indexes of each file from the index and as well as remove the entry from the
+ // metadata file.
+
+ if (sfm.size() > pathsFromFileList.size()) {
+ sfm.removeAll(pathsFromFileList);
+
+ for (String s : sfm) {
+ metadataMap.remove(s);
+ indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, s));
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.log(Level.DEBUG, "Index of the deleted file " + s + " was deleted from the index!");
+ }
+ }
+ }
+ }
+
+ /**
+ * Delete all indexes in the given directory.
+ * This will also remove the existing metadata file.
+ * It will be created when recreating the index.
+ * When deleting indexes, if any error occurred, the process will be rolled back and all the indexes will be
+ * restored.
+ * Otherwise the changes will be committed.
+ *
+ * @throws SystemException
+ * : An attempt to divide by zero
+ */
+ public void deleteAllIndexes() throws SystemException {
+ try {
+ indexWriter.deleteAll();
+ indexWriter.commit();
+ indexWriter.close();
+ metaFileUtil.deleteMetaDataFile();
+
+ for (File f : (new File(indexFolder)).listFiles())
+ Files.delete(f.toPath());
+
+ sb.finish();
+ result.set(abvs);
+ } catch (IOException e) {
+ try {
+ indexWriter.rollback();
+ indexWriter.close();
+
+ sb.finish();
+ result.set(abvs);
+ } catch (IOException e1) {
+ throw new SystemException(ErrorCode.FOAR0001, e1);
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java
new file mode 100644
index 0000000..5f41355
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java
@@ -0,0 +1,205 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index.update;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.xml.bind.DatatypeConverter;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * Utility class for writing, reading metadata file and generating checksum.
+ */
+public class MetaFileUtil {
+
+ private File metaFile;
+ private static final Logger LOGGER = Logger.getLogger("MetadataFileUtil");
+ private String index;
+ private String collection;
+ private ConcurrentHashMap<String, XmlMetadata> indexMap = new ConcurrentHashMap<>();
+
+ public MetaFileUtil(String indexFolder) {
+ this.metaFile = new File(indexFolder + "/" + Constants.META_FILE_NAME);
+ }
+
+ /**
+ * Checks for existing metadata file.
+ *
+ * @return true if the metadata file is present
+ */
+ public boolean isMetaFilePresent() {
+ return metaFile.exists();
+ }
+
+ /**
+ * Update the content of the metadata map.
+ * If the current collection data is present, replace it.
+ * Otherwise insert new.
+ *
+ * @param metadataMap
+ * : Set of XmlMetaData objects.
+ * @param index
+ * : The path to index location.
+ */
+ public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) {
+ this.indexMap = metadataMap;
+ this.index = index;
+
+ }
+
+ /**
+ * Method to get the set of xml metadata for a given collection
+ *
+ * @return : Map containing the set of XmlMetadata objects.\
+ */
+ public ConcurrentHashMap<String, XmlMetadata> getMetadata() {
+ return this.indexMap;
+ }
+
+ /**
+ * Read the metadata file and create an in-memory map containing collection paths and xml files.
+ */
+ public void readMetadataFile() {
+ try {
+ JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
+ Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
+ XmlMetadataCollection indexes = (XmlMetadataCollection) jaxbUnmarshaller.unmarshal(metaFile);
+
+ this.collection = indexes.getCollection();
+ this.index = indexes.getIndexLocation();
+
+ for (XmlMetadata metadata : indexes.getMetadataList()) {
+ this.indexMap.put(index, metadata);
+ }
+ } catch (JAXBException e) {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e);
+ }
+ }
+ }
+
+ /**
+ * Write the content of the ConcurrentHashMap to the xml metadata file.
+ */
+ public void writeMetadataToFile() {
+ XmlMetadataCollection xmlMetadataCollection = new XmlMetadataCollection();
+ List<XmlMetadata> metadataList = new ArrayList<>();
+
+ for (Map.Entry<String, XmlMetadata> entry : this.indexMap.entrySet()) {
+ metadataList.add(entry.getValue());
+ }
+
+ xmlMetadataCollection.setMetadataList(metadataList);
+ xmlMetadataCollection.setCollection(collection);
+ xmlMetadataCollection.setIndexLocation(this.index);
+ try {
+ FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile);
+ JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
+ Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
+ jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+ jaxbMarshaller.marshal(xmlMetadataCollection, fileOutputStream);
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!");
+ }
+ } catch (JAXBException | FileNotFoundException e) {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e);
+ }
+ }
+
+ }
+
+ /**
+ * Generate MD5 checksum string for a given file.
+ *
+ * @param file
+ * : File which the checksum should be generated.
+ * @return : Checksum String
+ * @throws IOException
+ * : The file is not available
+ */
+ public String generateMD5(File file) throws IOException {
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update(Files.readAllBytes(file.toPath()));
+ byte[] md5 = md.digest();
+ return DatatypeConverter.printHexBinary(md5);
+ } catch (NoSuchAlgorithmException e) {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.log(Level.ERROR, "No Such Algorithm Error " + e.getMessage());
+ }
+ return null;
+ }
+ }
+
+ /**
+ * Delete the existing Metadata file.
+ *
+ * @return True if deleted, false otherwise.
+ */
+ public boolean deleteMetaDataFile() {
+ try {
+ Files.delete(Paths.get(metaFile.getCanonicalPath()));
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.log(Level.DEBUG, "Metadata file deleted!");
+ }
+ return true;
+ } catch (IOException e) {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.log(Level.ERROR, "Metadata file could not be deleted!");
+ }
+ return false;
+ }
+ }
+
+ /**
+ * Get the collection for a given index location.
+ *
+ * @return collection folder for a given index.
+ */
+ public String getCollection() {
+ return this.collection;
+ }
+
+ /**
+ * Set the entry for given index and collection.
+ *
+ * @param collection
+ * : path to corresponding collection
+ */
+ public void setCollection(String collection) {
+ this.collection = collection;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java
new file mode 100644
index 0000000..fc96763
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java
@@ -0,0 +1,42 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index.update;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.List;
+
+/**
+ * Class for storing metadata information for vxquery index.
+ */
+@XmlAccessorType(XmlAccessType.PROPERTY)
+@XmlRootElement(name = "indexes")
+public class VXQueryIndex {
+
+ private List<XmlMetadataCollection> indexes;
+
+ public List<XmlMetadataCollection> getIndex() {
+ return indexes;
+ }
+
+ @XmlElement(name = "index", type = XmlMetadataCollection.class)
+ public void setIndex(List<XmlMetadataCollection> index) {
+ this.indexes = index;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java
new file mode 100644
index 0000000..063120c
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java
@@ -0,0 +1,74 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index.update;
+
+import java.io.Serializable;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Class to store metadata related to an XML file.
+ * This contains
+ * - Path to the xml file
+ * - MD5 Checksum String
+ * - File name
+ * - Last modified date
+ */
+@XmlRootElement(name = "file")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class XmlMetadata implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private String path;
+ private String md5;
+ private String fileName;
+ private String lastModified;
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public String getMd5() {
+ return md5;
+ }
+
+ public void setMd5(String md5) {
+ this.md5 = md5;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+
+ public String getLastModified() {
+ return lastModified;
+ }
+
+ public void setLastModified(String lastModified) {
+ this.lastModified = lastModified;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java
new file mode 100644
index 0000000..a1ca776
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java
@@ -0,0 +1,66 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index.update;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.List;
+
+/**
+ * Class for holding the collection information and the list of XML metadata related to the xml files in the
+ * collection.
+ */
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlRootElement(name = "index")
+public class XmlMetadataCollection {
+
+ @XmlAttribute(name = "location")
+ private String indexLocation;
+
+ @XmlAttribute(name = "collection")
+ private String collection;
+
+ @XmlElement(name = "file", type = XmlMetadata.class)
+ private List<XmlMetadata> metadataList;
+
+ public List<XmlMetadata> getMetadataList() {
+ return metadataList;
+ }
+
+ public void setMetadataList(List<XmlMetadata> metadataList) {
+ this.metadataList = metadataList;
+ }
+
+ public String getIndexLocation() {
+ return indexLocation;
+ }
+
+ public void setIndexLocation(String indexLocation) {
+ this.indexLocation = indexLocation;
+ }
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public void setCollection(String collection) {
+ this.collection = collection;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
deleted file mode 100644
index 2a45747..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
-
-/**
- * Constants used in updating index
- */
-public class Constants {
- public static String FIELD_PATH = "path";
- public static String META_FILE_NAME = "vxquery_index.xml";
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
deleted file mode 100644
index d3b9fdf..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
-
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.index.IndexDocumentBuilder;
-import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer;
-import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.text.SimpleDateFormat;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Update the index if the source files are changed.
- */
-public class IndexUpdater {
- private MetaFileUtil metaFileUtil;
- private ConcurrentHashMap<String, XmlMetadata> metadataMap;
- private IPointable result;
- private ByteBufferInputStream bbis;
- private DataInputStream di;
- private SequenceBuilder sb;
- private ArrayBackedValueStorage abvs;
- private ITreeNodeIdProvider nodeIdProvider;
- private ArrayBackedValueStorage abvsFileNode;
- private TaggedValuePointable nodep;
- private String nodeId;
- private IndexWriter indexWriter;
- private Set<String> pathsFromFileList;
- private String collectionFolder;
- private String indexFolder;
- private Logger LOGGER = Logger.getLogger("Index Updater");
- private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
- private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
-
- public IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp, ByteBufferInputStream bbis,
- DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider,
- ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep, String nodeId) {
- this.indexFolder = indexFolder;
- this.result = result;
- this.bbis = bbis;
- this.di = di;
- this.sb = sb;
- this.abvs = abvs;
- this.nodeIdProvider = nodeIdProvider;
- this.abvsFileNode = abvsFileNode;
- this.nodep = nodep;
- this.nodeId = nodeId;
- this.pathsFromFileList = new HashSet<>();
- }
-
- /**
- * Perform the initial configuration for index update/ delete processes.
- *
- * @throws SystemException
- * : If getting the index folder generates {@link SystemException}
- * @throws IOException
- * : If getting the index folder generates {@link IOException}
- */
- public void setup() throws SystemException, IOException {
-
- // Read the metadata file and load the metadata map into memory.
- metaFileUtil = new MetaFileUtil(indexFolder);
- metaFileUtil.readMetadataFile();
- metadataMap = metaFileUtil.getMetadata();
-
- // Retrieve the collection folder path.
- // Remove the entry for ease of the next steps.
- collectionFolder = metaFileUtil.getCollection();
-
- abvs.reset();
- sb.reset(abvs);
-
- Directory fsdir = FSDirectory.open(Paths.get(indexFolder));
- indexWriter = new IndexWriter(fsdir, new IndexWriterConfig(new CaseSensitiveAnalyzer())
- .setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND));
- }
-
- /**
- * Wrapper for update index function.
- *
- * @throws IOException
- * : If the directory doesn't exist
- */
- public void updateIndex() throws IOException {
- File collectionDirectory = new File(collectionFolder);
- if (!collectionDirectory.exists()) {
- throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist.");
- }
-
- //Execute update index process
- updateIndex(collectionDirectory);
-
- //Detect deleted files and execute the delete index process.
- deleteIndexOfDeletedFiles(metadataMap.keySet(), pathsFromFileList);
- updateMetadataFile();
- }
-
- /**
- * Close opened IndexWriter and terminate the index update/ delete process.
- *
- * @throws IOException
- * : If exiting the index folder generates {@link IOException}
- */
- public void exit() throws IOException {
- indexWriter.forceMerge(1);
-
- indexWriter.close();
-
- sb.finish();
- result.set(abvs);
- }
-
- /**
- * Functional wrapper to update Metadata file.
- *
- * @throws IOException
- * : If updating metadata folder generates {@link IOException}
- */
- public synchronized void updateMetadataFile() throws IOException {
- //Write the updated metadata to the file.
- metaFileUtil.updateMetadataMap(metadataMap, indexFolder);
- metaFileUtil.writeMetadataToFile();
- }
-
- /**
- * Check the collection for changes.
- * If changes are detected, update the index
- *
- * @param collection
- * : Collection folder path
- */
- private void updateIndex(File collection) throws IOException {
-
- File[] list = collection.listFiles();
-
- assert list != null;
- for (File file : list) {
- pathsFromFileList.add(file.getCanonicalPath());
- if (indexConstructorUtil.readableXmlFile(file.getCanonicalPath())) {
- XmlMetadata data = metadataMap.get(file.getCanonicalPath());
- String md5 = metaFileUtil.generateMD5(file);
-
- abvsFileNode.reset();
-
- IndexDocumentBuilder indexDocumentBuilder;
- if (data != null) {
-
- // This case checks whether the file has been changed.
- // If the file has changed, delete the existing document, create a new index document and add it
- // to the current index.
- // At the same time, update the metadata for the file.
- if (!md5.equals(data.getMd5())) {
-
- //Update index corresponding to the xml file.
- indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath()));
- indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep,
- abvsFileNode, nodeIdProvider, bbis, di, nodeId);
- indexDocumentBuilder.printStart();
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.log(Level.DEBUG, "New Index is created for updated file " + file.getCanonicalPath());
- }
-
- //Update the metadata map.
- XmlMetadata metadata = updateEntry(file, data);
- metadataMap.replace(file.getCanonicalPath(), metadata);
-
- }
- } else {
-
- // In this case, the xml file has not added to the index. (It is a newly added file)
- // Therefore generate a new index for this file and add it to the existing index.
- indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep, abvsFileNode,
- nodeIdProvider, bbis, di, nodeId);
- indexDocumentBuilder.printStart();
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.log(Level.DEBUG, "New Index is created for newly added file " + file.getCanonicalPath());
- }
-
- XmlMetadata metadata = updateEntry(file, null);
- metadataMap.put(file.getCanonicalPath(), metadata);
- }
- } else if (file.isDirectory()) {
- updateIndex(file);
- }
- }
- }
-
- /**
- * Update the current XmlMetadata object related to the currently reading XML file.
- *
- * @param file
- * : XML file
- * @param metadata
- * : Existing metadata object
- * @return : XML metadata object with updated fields.
- * @throws IOException
- * : If getting the file info generates {@link IOException}
- */
- private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException {
-
- if (metadata == null) {
- metadata = new XmlMetadata();
- }
- metadata.setFileName(file.getName());
- metadata.setPath(file.getCanonicalPath());
- metadata.setMd5(metaFileUtil.generateMD5(file));
- metadata.setLastModified(sdf.format(file.lastModified()));
- return metadata;
- }
-
- /**
- * Delete the index of deleted files.
- *
- * @param pathsFromMap
- * : Set of paths taken from metafile.
- * @param pathsFromFileList
- * : Set of paths taken from list of existing files.
- * @throws IOException
- * : If deleting Documents generates {@link IOException}
- */
- private void deleteIndexOfDeletedFiles(Set<String> pathsFromMap, Set<String> pathsFromFileList) throws IOException {
- Set<String> sfm = new HashSet<>(pathsFromMap);
-
- // If any file has been deleted from the collection, the number of files stored in metadata is higher than
- // the actual number of files.
- // With set difference, the paths of deleted files are taken from the stored metadata.
- // Delete the corresponding indexes of each file from the index and as well as remove the entry from the
- // metadata file.
-
- if (sfm.size() > pathsFromFileList.size()) {
- sfm.removeAll(pathsFromFileList);
-
- for (String s : sfm) {
- metadataMap.remove(s);
- indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, s));
- if (LOGGER.isDebugEnabled()) {
- LOGGER.log(Level.DEBUG, "Index of the deleted file " + s + " was deleted from the index!");
- }
- }
- }
- }
-
- /**
- * Delete all indexes in the given directory.
- * This will also remove the existing metadata file.
- * It will be created when recreating the index.
- * When deleting indexes, if any error occurred, the process will be rolled back and all the indexes will be
- * restored.
- * Otherwise the changes will be committed.
- *
- * @throws SystemException
- * : An attempt to divide by zero
- */
- public void deleteAllIndexes() throws SystemException {
- try {
- indexWriter.deleteAll();
- indexWriter.commit();
- indexWriter.close();
- metaFileUtil.deleteMetaDataFile();
-
- for (File f : (new File(indexFolder)).listFiles())
- Files.delete(f.toPath());
-
- sb.finish();
- result.set(abvs);
- } catch (IOException e) {
- try {
- indexWriter.rollback();
- indexWriter.close();
-
- sb.finish();
- result.set(abvs);
- } catch (IOException e1) {
- throw new SystemException(ErrorCode.FOAR0001);
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
deleted file mode 100644
index 0dfb54a..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
-
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import javax.xml.bind.DatatypeConverter;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
-import javax.xml.bind.Unmarshaller;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Utility class for writing, reading metadata file and generating checksum.
- */
-public class MetaFileUtil {
-
- private File metaFile;
- private Logger LOGGER = Logger.getLogger("MetadataFileUtil");
- private String index;
- private String collection;
- private ConcurrentHashMap<String, XmlMetadata> indexMap = new ConcurrentHashMap<>();
-
- public MetaFileUtil(String indexFolder) {
- this.metaFile = new File(indexFolder + "/" + Constants.META_FILE_NAME);
- }
-
- /**
- * Checks for existing metadata file.
- *
- * @return true if the metadata file is present
- */
- public boolean isMetaFilePresent() {
- return metaFile.exists();
- }
-
- /**
- * Update the content of the metadata map.
- * If the current collection data is present, replace it.
- * Otherwise insert new.
- * @param metadataMap : Set of XmlMetaData objects.
- * @param index : The path to index location.
- */
- public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) {
- this.indexMap = metadataMap;
- this.index = index;
-
- }
-
- /**
- * Method to get the set of xml metadata for a given collection
- *
- * @return : Map containing the set of XmlMetadata objects.\
- */
- public ConcurrentHashMap<String, XmlMetadata> getMetadata() {
- return this.indexMap;
- }
-
- /**
- * Read the metadata file and create an in-memory map containing collection paths and xml files.
- */
- public void readMetadataFile() {
- try {
- JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
- Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
- XmlMetadataCollection indexes = (XmlMetadataCollection) jaxbUnmarshaller.unmarshal(metaFile);
-
- this.collection = indexes.getCollection();
- this.index = indexes.getIndexLocation();
-
- for (XmlMetadata metadata : indexes.getMetadataList()) {
- this.indexMap.put(index, metadata);
- }
- } catch (JAXBException e) {
- if (LOGGER.isTraceEnabled()) {
- LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e);
- }
- }
- }
-
- /**
- * Write the content of the ConcurrentHashMap to the xml metadata file.
- */
- public void writeMetadataToFile() {
- XmlMetadataCollection collection = new XmlMetadataCollection();
- List<XmlMetadata> metadataList = new ArrayList<>();
-
- for (Map.Entry<String, XmlMetadata> entry : this.indexMap.entrySet()) {
- metadataList.add(entry.getValue());
- }
-
- collection.setMetadataList(metadataList);
- collection.setCollection(this.collection);
- collection.setIndexLocation(this.index);
- try{
- FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile);
- JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
- Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
- jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
- jaxbMarshaller.marshal(collection, fileOutputStream);
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!");
- }
- } catch (JAXBException | FileNotFoundException e) {
- if (LOGGER.isTraceEnabled()) {
- LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e);
- }
- }
-
-
- }
-
-
- /**
- * Generate MD5 checksum string for a given file.
- *
- * @param file : File which the checksum should be generated.
- * @return : Checksum String
- * @throws IOException : The file is not available
- */
- public String generateMD5(File file) throws IOException {
- try {
- MessageDigest md = MessageDigest.getInstance("MD5");
- md.update(Files.readAllBytes(file.toPath()));
- byte[] md5 = md.digest();
- return DatatypeConverter.printHexBinary(md5);
- } catch (NoSuchAlgorithmException e) {
- if (LOGGER.isTraceEnabled()) {
- LOGGER.log(Level.ERROR, "No Such Algorithm Error " + e.getMessage());
- }
- return null;
- }
- }
-
- /**
- * Delete the existing Metadata file.
- *
- * @return True if deleted, false otherwise.
- */
- public boolean deleteMetaDataFile() {
- try {
- Files.delete(Paths.get(metaFile.getCanonicalPath()));
- if (LOGGER.isDebugEnabled()){
- LOGGER.log(Level.DEBUG, "Metadata file deleted!");
- }
- return true;
- } catch (IOException e) {
- if (LOGGER.isTraceEnabled()){
- LOGGER.log(Level.ERROR, "Metadata file could not be deleted!");
- }
- return false;
- }
- }
-
- /**
- * Get the collection for a given index location.
- * @return collection folder for a given index.
- */
- public String getCollection() {
- return this.collection;
- }
-
- /**
- * Set the entry for given index and collection.
- * @param collection : path to corresponding collection
- */
- public void setCollection(String collection) {
- this.collection = collection;
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java
deleted file mode 100644
index fa92b2f..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.util.List;
-
-/**
- * Class for storing metadata information for vxquery index.
- */
-@XmlAccessorType(XmlAccessType.PROPERTY)
-@XmlRootElement(name = "indexes")
-public class VXQueryIndex {
-
- private List<XmlMetadataCollection> indexes;
-
- public List<XmlMetadataCollection> getIndex() {
- return indexes;
- }
-
- @XmlElement(name = "index", type = XmlMetadataCollection.class)
- public void setIndex(List<XmlMetadataCollection> index) {
- this.indexes = index;
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java
deleted file mode 100644
index b6da6d9..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
-
-/**
- * Class to store metadata related to an XML file.
- * This contains
- * - Path to the xml file
- * - MD5 Checksum String
- * - File name
- * - Last modified date
- */
-@XmlRootElement(name = "file")
-@XmlAccessorType(XmlAccessType.FIELD)
-public class XmlMetadata implements Serializable {
-
- private String path;
- private String md5;
- private String fileName;
- private String lastModified;
-
- public String getPath() {
- return path;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public String getMd5() {
- return md5;
- }
-
- public void setMd5(String md5) {
- this.md5 = md5;
- }
-
- public String getFileName() {
- return fileName;
- }
-
- public void setFileName(String fileName) {
- this.fileName = fileName;
- }
-
- public String getLastModified() {
- return lastModified;
- }
-
- public void setLastModified(String lastModified) {
- this.lastModified = lastModified;
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
deleted file mode 100644
index 1f5c3e9..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlAttribute;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.util.List;
-
-/**
- * Class for holding the collection information and the list of XML metadata related to the xml files in the
- * collection.
- */
-@XmlAccessorType(XmlAccessType.FIELD)
-@XmlRootElement(name = "index")
-public class XmlMetadataCollection {
-
- @XmlAttribute(name = "location")
- private String indexLocation;
-
- @XmlAttribute(name = "collection")
- private String collection;
-
- @XmlElement(name = "file", type = XmlMetadata.class)
- private List<XmlMetadata> metadataList;
-
- public List<XmlMetadata> getMetadataList() {
- return metadataList;
- }
-
- public void setMetadataList(List<XmlMetadata> metadataList) {
- this.metadataList = metadataList;
- }
-
- public String getIndexLocation() {
- return indexLocation;
- }
-
- public void setIndexLocation(String indexLocation) {
- this.indexLocation = indexLocation;
- }
-
- public String getCollection() {
- return collection;
- }
-
- public void setCollection(String collection) {
- this.collection = collection;
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java
index 665c812..85ef4ca 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java
@@ -64,7 +64,7 @@ public class JnDocScalarEvaluatorFactory extends AbstractTaggedValueArgumentScal
tvp.getValue(stringp);
try {
IParser parser = new JSONParser();
- FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser);
+ FunctionHelper.readInDocFromPointable(stringp, abvs, parser);
} catch (IOException e) {
throw new SystemException(ErrorCode.FODC0002, e);
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
index db908f6..15fd624 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
@@ -78,7 +78,7 @@ public class FnDocAvailableScalarEvaluatorFactory extends AbstractTaggedValueArg
tvp.getValue(stringp);
try {
IParser parser = new XMLParser(false, nodeIdProvider, nodeId);
- FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser);
+ FunctionHelper.readInDocFromPointable(stringp, abvs, parser);
XDMConstants.setTrue(result);
} catch (Exception e) {
XDMConstants.setFalse(result);
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java
index 5f08a8e..2fd1755 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java
@@ -79,7 +79,7 @@ public class FnDocScalarEvaluatorFactory extends AbstractTaggedValueArgumentScal
try {
// Only one document should be parsed so its ok to have a unique parser.
IParser parser = new XMLParser(false, nodeIdProvider, nodeId);
- FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser);
+ FunctionHelper.readInDocFromPointable(stringp, abvs, parser);
} catch (Exception e) {
throw new SystemException(ErrorCode.SYSE0001, e);
}
[3/3] vxquery git commit: [VXQUERY-196][VXQUERY-204][VXQUERY-228]
Cleaning up indexing query statements
Posted by ch...@apache.org.
[VXQUERY-196][VXQUERY-204][VXQUERY-228] Cleaning up indexing query statements
1) Move XPath out of arguments for collection-from-index, leave only the collection path.
2) Hide the index usage from the user by creating a rewrite rule to figure out the existence of index.
3) Remove the unnecessary header tags that collection-from-index creates.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/4a38f670
Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/4a38f670
Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/4a38f670
Branch: refs/heads/master
Commit: 4a38f670cd11231bd18a321cbacdd4f4e109b50a
Parents: 37642be
Author: Christina Pavlopoulou <cp...@ucr.edu>
Authored: Tue May 30 11:54:31 2017 -0700
Committer: Christina Pavlopoulou <cp...@ucr.edu>
Committed: Tue Jul 11 10:20:22 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/vxquery/cli/VXQuery.java | 141 ++++----
.../apache/vxquery/common/VXQueryCommons.java | 3 +-
.../rewriter/rules/IntroduceCollectionRule.java | 30 +-
.../rewriter/rules/IntroduceIndexingRule.java | 10 +-
.../rules/PushChildIntoDataScanRule.java | 7 +-
.../rules/PushValueIntoDatascanRule.java | 3 +
.../vxquery/functions/builtin-functions.xml | 3 +-
.../vxquery/index/IndexDocumentBuilder.java | 14 +-
.../VXQueryCollectionOperatorDescriptor.java | 58 ++--
.../metadata/VXQueryIndexingDataSource.java | 16 +-
.../VXQueryIndexingOperatorDescriptor.java | 181 ++++++-----
.../metadata/VXQueryMetadataProvider.java | 34 +-
.../functions/index/IndexConstructorUtil.java | 69 ++--
.../index/ShowIndexScalarEvaluatorFactory.java | 67 ----
.../ShowIndexesScalarEvaluatorFactory.java | 67 ++++
.../functions/index/VXQueryIndexReader.java | 115 +++++--
.../index/centralizer/IndexCentralizerUtil.java | 175 ++++++++++
.../index/centralizer/IndexDirectory.java | 43 +++
.../index/centralizer/IndexLocator.java | 52 +++
.../indexCentralizer/IndexCentralizerUtil.java | 165 ----------
.../index/indexCentralizer/IndexDirectory.java | 42 ---
.../index/indexCentralizer/IndexLocator.java | 50 ---
.../functions/index/update/Constants.java | 28 ++
.../functions/index/update/IndexUpdater.java | 304 ++++++++++++++++++
.../functions/index/update/MetaFileUtil.java | 205 ++++++++++++
.../functions/index/update/VXQueryIndex.java | 42 +++
.../functions/index/update/XmlMetadata.java | 74 +++++
.../index/update/XmlMetadataCollection.java | 66 ++++
.../functions/index/updateIndex/Constants.java | 25 --
.../index/updateIndex/IndexUpdater.java | 319 -------------------
.../index/updateIndex/MetaFileUtil.java | 198 ------------
.../index/updateIndex/VXQueryIndex.java | 42 ---
.../index/updateIndex/XmlMetadata.java | 72 -----
.../updateIndex/XmlMetadataCollection.java | 66 ----
.../json/JnDocScalarEvaluatorFactory.java | 2 +-
.../FnDocAvailableScalarEvaluatorFactory.java | 2 +-
.../node/FnDocScalarEvaluatorFactory.java | 2 +-
.../runtime/functions/util/FunctionHelper.java | 21 +-
.../vxquery/xmlparser/SAXContentHandler.java | 7 +-
.../xmlquery/query/XMLQueryCompiler.java | 4 +-
.../vxquery/indexing/MetaFileUtilTest.java | 4 +-
.../apache/vxquery/indexing/TestConstants.java | 4 +-
.../xmlquery/query/SimpleXQueryTest.java | 2 +-
.../org/apache/vxquery/xtest/TestRunner.java | 56 +++-
.../Indexing/Partition-1/useIndex1_user.txt | 2 +
.../Indexing/Partition-2/useIndex1_user.txt | 2 +
.../Indexing/Partition-4/useIndex1_user.txt | 2 +
.../XQuery/Indexing/Partition-1/useIndex1.xq | 4 +-
.../Indexing/Partition-1/useIndex1_user.xq | 25 ++
.../XQuery/Indexing/Partition-1/useIndex2.xq | 2 +-
.../XQuery/Indexing/Partition-1/useIndex3.xq | 2 +-
.../XQuery/Indexing/Partition-1/useIndex4.xq | 2 +-
.../XQuery/Indexing/Partition-1/useIndex5.xq | 2 +-
.../XQuery/Indexing/Partition-1/useIndex6.xq | 2 +-
.../XQuery/Indexing/Partition-1/useIndex7.xq | 4 +-
.../XQuery/Indexing/Partition-2/useIndex1.xq | 2 +-
.../Indexing/Partition-2/useIndex1_user.xq | 25 ++
.../XQuery/Indexing/Partition-2/useIndex2.xq | 2 +-
.../XQuery/Indexing/Partition-2/useIndex3.xq | 2 +-
.../XQuery/Indexing/Partition-2/useIndex4.xq | 2 +-
.../XQuery/Indexing/Partition-2/useIndex5.xq | 2 +-
.../XQuery/Indexing/Partition-2/useIndex6.xq | 2 +-
.../XQuery/Indexing/Partition-2/useIndex7.xq | 4 +-
.../XQuery/Indexing/Partition-4/useIndex1.xq | 2 +-
.../Indexing/Partition-4/useIndex1_user.xq | 25 ++
.../XQuery/Indexing/Partition-4/useIndex2.xq | 2 +-
.../XQuery/Indexing/Partition-4/useIndex3.xq | 2 +-
.../XQuery/Indexing/Partition-4/useIndex4.xq | 2 +-
.../XQuery/Indexing/Partition-4/useIndex5.xq | 2 +-
.../XQuery/Indexing/Partition-4/useIndex6.xq | 2 +-
.../XQuery/Indexing/Partition-4/useIndex7.xq | 4 +-
.../src/test/resources/VXQueryCatalog.xml | 4 +-
.../src/test/resources/cat/IndexingQueries.xml | 15 +
73 files changed, 1656 insertions(+), 1384 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
----------------------------------------------------------------------
diff --git a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
index e0e3843..25ff9c4 100644
--- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
+++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
@@ -14,6 +14,7 @@
*/
package org.apache.vxquery.cli;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -23,6 +24,7 @@ import java.io.StringReader;
import java.net.InetAddress;
import java.nio.file.Files;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
@@ -66,12 +68,13 @@ import org.kohsuke.args4j.Option;
public class VXQuery {
private final CmdLineOptions opts;
+ private final CmdLineOptions indexOpts;
private ClusterControllerService cc;
private NodeControllerService[] ncs;
private IHyracksClientConnection hcc;
private IHyracksDataset hds;
-
+ private List<String> collectionList;
private ResultSetId resultSetId;
private static List<String> timingMessages = new ArrayList<>();
private static long sumTiming;
@@ -87,6 +90,16 @@ public class VXQuery {
*/
public VXQuery(CmdLineOptions opts) {
this.opts = opts;
+ // The index query returns only the result, without any other information.
+ this.indexOpts = opts;
+ indexOpts.showAST = false;
+ indexOpts.showOET = false;
+ indexOpts.showQuery = false;
+ indexOpts.showRP = false;
+ indexOpts.showTET = false;
+ indexOpts.timing = false;
+ indexOpts.compileOnly = false;
+ this.collectionList = new ArrayList<String>();
}
/**
@@ -168,71 +181,87 @@ public class VXQuery {
* @throws SystemException
* @throws Exception
*/
+
private void runQueries() throws Exception {
- Date start;
- Date end;
- for (String query : opts.arguments) {
- String qStr = slurp(query);
- if (opts.showQuery) {
- System.err.println(qStr);
+ List<String> queries = opts.arguments;
+ // Run the showIndexes query before executing any target query, to store the index metadata
+ List<String> queriesIndex = new ArrayList<String>();
+ queriesIndex.add("vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndexes.xq");
+ OutputStream resultStream = new ByteArrayOutputStream();
+ executeQuery(queriesIndex.get(0), 1, resultStream, indexOpts);
+ ByteArrayOutputStream bos = (ByteArrayOutputStream) resultStream;
+ String result = new String(bos.toByteArray());
+ String[] collections = result.split("\n");
+ this.collectionList = Arrays.asList(collections);
+ executeQueries(queries);
+ }
+
+ public void executeQueries(List<String> queries) throws Exception {
+ for (String query : queries) {
+ OutputStream resultStream = System.out;
+ if (opts.resultFile != null) {
+ resultStream = new FileOutputStream(new File(opts.resultFile));
}
+ executeQuery(query, opts.repeatExec, resultStream, opts);
+ }
+ }
- VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET,
- opts.showOET, opts.showRP);
+ public void executeQuery(String query, int repeatedExecution, OutputStream resultStream, CmdLineOptions options)
+ throws Exception {
+ PrintWriter writer = new PrintWriter(resultStream, true);
+ String qStr = slurp(query);
+ if (opts.showQuery) {
+ writer.println(qStr);
+ }
+ VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, opts.showOET,
+ opts.showRP);
- start = opts.timing ? new Date() : null;
+ Date start = opts.timing ? new Date() : null;
- Map<String, NodeControllerInfo> nodeControllerInfos = null;
- if (hcc != null) {
- nodeControllerInfos = hcc.getNodeControllerInfos();
- }
- XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize,
- opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf);
- resultSetId = createResultSetId();
- CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
- resultSetId, null);
- compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel);
- // if -timing argument passed, show the starting and ending times
- if (opts.timing) {
- end = new Date();
- timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms");
- }
- if (opts.compileOnly) {
- continue;
- }
-
- Module module = compiler.getModule();
- JobSpecification js = module.getHyracksJobSpecification();
+ Map<String, NodeControllerInfo> nodeControllerInfos = null;
+ if (hcc != null) {
+ nodeControllerInfos = hcc.getNodeControllerInfos();
+ }
+ XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize,
+ opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf);
+ resultSetId = createResultSetId();
+ CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
+ resultSetId, null);
+ compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel, this.collectionList);
+ // if -timing argument passed, show the starting and ending times
+ Date end = opts.timing ? new Date() : null;
+ if (opts.timing) {
+ timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms");
+ }
+ if (opts.compileOnly) {
+ return;
+ }
- DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext());
- js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
+ Module module = compiler.getModule();
+ JobSpecification js = module.getHyracksJobSpecification();
- OutputStream resultStream = System.out;
- if (opts.resultFile != null) {
- resultStream = new FileOutputStream(new File(opts.resultFile));
- }
+ DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext());
+ js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
- PrintWriter writer = new PrintWriter(resultStream, true);
- // Repeat execution for number of times provided in -repeatexec argument
- for (int i = 0; i < opts.repeatExec; ++i) {
- start = opts.timing ? new Date() : null;
- runJob(js, writer);
- // if -timing argument passed, show the starting and ending times
- if (opts.timing) {
- end = new Date();
- long currentRun = end.getTime() - start.getTime();
- if ((i + 1) > opts.timingIgnoreQueries) {
- sumTiming += currentRun;
- sumSquaredTiming += currentRun * currentRun;
- if (currentRun < minTiming) {
- minTiming = currentRun;
- }
- if (maxTiming < currentRun) {
- maxTiming = currentRun;
- }
+ // Repeat execution for number of times provided in -repeatexec argument
+ for (int i = 0; i < repeatedExecution; ++i) {
+ start = opts.timing ? new Date() : null;
+ runJob(js, writer);
+ // if -timing argument passed, show the starting and ending times
+ if (opts.timing) {
+ end = new Date();
+ long currentRun = end.getTime() - start.getTime();
+ if ((i + 1) > opts.timingIgnoreQueries) {
+ sumTiming += currentRun;
+ sumSquaredTiming += currentRun * currentRun;
+ if (currentRun < minTiming) {
+ minTiming = currentRun;
+ }
+ if (maxTiming < currentRun) {
+ maxTiming = currentRun;
}
- timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms");
}
+ timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms");
}
}
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java b/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
index ceaf3c7..400fb15 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
@@ -35,7 +35,8 @@ public class VXQueryCommons {
static {
indexingFunctions.add(BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier());
- indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier());
+ indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier());
+ indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_1.getFunctionIdentifier());
indexingFunctions.add(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier());
indexingFunctions.add(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier());
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
index 20283d8..11d3795 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
@@ -16,12 +16,16 @@
*/
package org.apache.vxquery.compiler.rewriter.rules;
+import java.util.ArrayList;
+
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.vxquery.common.VXQueryCommons;
import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext;
import org.apache.vxquery.metadata.VXQueryCollectionDataSource;
+import org.apache.vxquery.metadata.VXQueryIndexingDataSource;
+import org.apache.vxquery.metadata.VXQueryMetadataProvider;
import org.apache.vxquery.types.AnyItemType;
import org.apache.vxquery.types.Quantifier;
import org.apache.vxquery.types.SequenceType;
@@ -61,11 +65,35 @@ public class IntroduceCollectionRule extends AbstractCollectionRule {
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context;
String[] args = getFunctionalArguments(opRef, VXQueryCommons.collectionFunctions);
-
+ VXQueryMetadataProvider metadata = (VXQueryMetadataProvider) context.getMetadataProvider();
if (args != null) {
String collectionName = args[0];
// Build the new operator and update the query plan.
int collectionId = vxqueryContext.newCollectionId();
+ ArrayList<String> collectionTempName = new ArrayList<String>();
+ collectionTempName.add(collectionName);
+ if (collectionName.contains("|")) {
+ collectionTempName.remove(0);
+ int index = collectionName.indexOf("|");
+ int start = 0;
+ while (index >= 0) {
+ collectionTempName.add(collectionName.substring(start, index));
+ start = index + 1;
+ index = collectionName.indexOf("|", index + 1);
+ if (index == -1) {
+ collectionTempName.add(collectionName.substring(start));
+ }
+ }
+ }
+ if (metadata.hasIndex(collectionTempName)) {
+ VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collectionName,
+ SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR),
+ functionCall.getFunctionIdentifier().getName());
+ if (ids != null) {
+ ids.setTotalDataSources(vxqueryContext.getTotalDataSources());
+ return setDataSourceScan(ids, opRef);
+ }
+ }
VXQueryCollectionDataSource ds = VXQueryCollectionDataSource.create(collectionId, collectionName,
SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR));
if (ds != null) {
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
index 5b96131..6e60d75 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
@@ -33,19 +33,19 @@ import org.apache.vxquery.types.SequenceType;
public class IntroduceIndexingRule extends AbstractCollectionRule {
@Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context;
String args[] = getFunctionalArguments(opRef, VXQueryCommons.indexingFunctions);
if (args != null) {
String collection = args[0];
- String elementPath = args.length > 1?args[1]:null;
-
// Build the new operator and update the query plan.
int collectionId = vxqueryContext.newCollectionId();
- VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collection, elementPath,
- SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR), functionCall.getFunctionIdentifier().getName());
+ VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collection,
+ SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR),
+ functionCall.getFunctionIdentifier().getName());
if (ids != null) {
ids.setTotalDataSources(vxqueryContext.getTotalDataSources());
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
index dbcce54..2773154 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
@@ -55,12 +55,9 @@ public class PushChildIntoDataScanRule extends AbstractPushExpressionIntoDatasca
@Override
boolean updateDataSource(IVXQueryDataSource datasource, Mutable<ILogicalExpression> expression) {
- //TODO: indexing needs to be extended to support push child into datascan
- if (datasource.usingIndex()) {
- return false;
- }
- boolean added = false;
List<Mutable<ILogicalExpression>> finds = new ArrayList<Mutable<ILogicalExpression>>();
+ boolean added = false;
+
ExpressionToolbox.findAllFunctionExpressions(expression, BuiltinOperators.CHILD.getFunctionIdentifier(), finds);
for (int i = finds.size(); i > 0; --i) {
int typeId = ExpressionToolbox.getTypeExpressionTypeArgument(finds.get(i - 1));
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java
index 1d8a55d..b901469 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java
@@ -59,6 +59,9 @@ public class PushValueIntoDatascanRule extends AbstractPushExpressionIntoDatasca
@Override
boolean updateDataSource(IVXQueryDataSource datasource, Mutable<ILogicalExpression> expression) {
+ if (datasource.usingIndex()) {
+ return false;
+ }
VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) datasource;
boolean added = false;
List<Mutable<ILogicalExpression>> finds = new ArrayList<Mutable<ILogicalExpression>>();
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
index d64f423..4932a78 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
+++ b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
@@ -152,7 +152,6 @@
<!-- fn:collection-from-index($indexfolder as xs:string?, $elementpath as xs:string?) as node()* -->
<function name="fn:collection-from-index">
<param name="index-folder" type="xs:string?"/>
- <param name="element-path" type="xs:string?"/>
<return type="node()*"/>
<property type="DocumentOrder" class="org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.InputPropertyPropagationPolicy">
<argument value="0"/>
@@ -166,7 +165,7 @@
<!-- fn:show-indexes as node()* -->
<function name="fn:show-indexes">
<return type="node()*"/>
- <runtime type="scalar" class="org.apache.vxquery.runtime.functions.index.ShowIndexScalarEvaluatorFactory"/>
+ <runtime type="scalar" class="org.apache.vxquery.runtime.functions.index.ShowIndexesScalarEvaluatorFactory"/>
</function>
<!-- fn:collection-with-tag($arg1 as xs:string?, $arg2 as xs:string?) as node()* -->
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java b/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java
index 7524da4..1df31dd 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.primitive.BooleanPointable;
import org.apache.hyracks.data.std.primitive.BytePointable;
import org.apache.hyracks.data.std.primitive.DoublePointable;
@@ -56,12 +55,10 @@ import org.apache.vxquery.datamodel.accessors.nodes.NodeTreePointable;
import org.apache.vxquery.datamodel.accessors.nodes.TextOrCommentNodePointable;
import org.apache.vxquery.datamodel.values.ValueTag;
import org.apache.vxquery.runtime.functions.cast.CastToStringOperation;
-import org.apache.vxquery.runtime.functions.index.updateIndex.Constants;
+import org.apache.vxquery.runtime.functions.index.update.Constants;
import org.apache.vxquery.serializer.XMLSerializer;
public class IndexDocumentBuilder extends XMLSerializer {
- private final IPointable treePointable;
-
private final PointablePool pp;
private NodeTreePointable ntp;
@@ -88,16 +85,11 @@ public class IndexDocumentBuilder extends XMLSerializer {
}
//TODO: Handle Processing Instructions, PrefixedNames, and Namepsace entries
- public IndexDocumentBuilder(IPointable tree, IndexWriter inWriter, String file) {
- this.treePointable = tree;
+ public IndexDocumentBuilder(TaggedValuePointable tvp, IndexWriter inWriter, String file) {
writer = inWriter;
this.filePath = file;
- //convert to tagged value pointable
- TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
- tvp.set(treePointable.getByteArray(), 0, treePointable.getLength());
-
//get bytes and info from doc pointer
bstart = tvp.getByteArray();
sstart = tvp.getStartOffset();
@@ -105,7 +97,7 @@ public class IndexDocumentBuilder extends XMLSerializer {
doc = new Document();
- results = new ArrayList<ComplexItem>();
+ results = new ArrayList<>();
pp = PointablePoolFactory.INSTANCE.createPointablePool();
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
index 623b48c..a3756d5 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
@@ -59,7 +59,6 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import org.apache.hyracks.hdfs.ContextFactory;
@@ -129,38 +128,14 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
fta.reset(buffer);
String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
- Reader input;
+
if (!collectionModifiedName.contains("hdfs:/")) {
File collectionDirectory = new File(collectionModifiedName);
// check if directory is in the local file system
if (collectionDirectory.exists()) {
// Go through each tuple.
if (collectionDirectory.isDirectory()) {
- for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) {
- Iterator<File> it = FileUtils.iterateFiles(collectionDirectory,
- new VXQueryIOFileFilter(), TrueFileFilter.INSTANCE);
- while (it.hasNext()) {
- File file = it.next();
- String fileName = file.getName().toLowerCase();
- if (fileName.endsWith(".xml")) {
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.fine("Starting to read XML document: " + file.getAbsolutePath());
- }
- parser.parseElements(file, writer, tupleIndex);
- } else if (fileName.endsWith(".json")) {
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.fine("Starting to read JSON document: " + file.getAbsolutePath());
- }
- try {
- jsonAbvs.reset();
- input = new InputStreamReader(new FileInputStream(file));
- jparser.parse(input, jsonAbvs, writer, appender);
- } catch (FileNotFoundException e) {
- throw new HyracksDataException(e.toString());
- }
- }
- }
- }
+ xmlAndJsonCollection(collectionDirectory);
} else {
throw new HyracksDataException("Invalid directory parameter (" + nodeId + ":"
+ collectionDirectory.getAbsolutePath() + ") passed to collection.");
@@ -272,6 +247,35 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO
}
}
+ public void xmlAndJsonCollection(File directory) throws HyracksDataException {
+ Reader input;
+ for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) {
+ Iterator<File> it = FileUtils.iterateFiles(directory, new VXQueryIOFileFilter(),
+ TrueFileFilter.INSTANCE);
+ while (it.hasNext()) {
+ File file = it.next();
+ String fileName = file.getName().toLowerCase();
+ if (fileName.endsWith(".xml")) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Starting to read XML document: " + file.getAbsolutePath());
+ }
+ parser.parseElements(file, writer, tupleIndex);
+ } else if (fileName.endsWith(".json")) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Starting to read JSON document: " + file.getAbsolutePath());
+ }
+ try {
+ jsonAbvs.reset();
+ input = new InputStreamReader(new FileInputStream(file));
+ jparser.parse(input, jsonAbvs, writer, appender);
+ } catch (FileNotFoundException e) {
+ throw new HyracksDataException(e.toString());
+ }
+ }
+ }
+ }
+ }
+
@Override
public void fail() throws HyracksDataException {
writer.fail();
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
index ea69cfd..d55530d 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
@@ -31,14 +31,11 @@ import org.apache.vxquery.compiler.rewriter.rules.CollectionFileDomain;
*/
public class VXQueryIndexingDataSource extends AbstractVXQueryDataSource {
- private String elementPath;
private String function;
- private VXQueryIndexingDataSource(int id, String collection, String elementPath, Object[] types,
- String functionCall) {
+ private VXQueryIndexingDataSource(int id, String collection, Object[] types, String functionCall) {
this.dataSourceId = id;
this.collectionName = collection;
- this.elementPath = elementPath;
this.function = functionCall;
this.collectionPartitions = collectionName.split(DELIMITER);
this.types = types;
@@ -56,13 +53,8 @@ public class VXQueryIndexingDataSource extends AbstractVXQueryDataSource {
this.valueSeq = new ArrayList<>();
}
- public static VXQueryIndexingDataSource create(int id, String collection, String index, Object type,
- String function) {
- return new VXQueryIndexingDataSource(id, collection, index, new Object[] { type }, function);
- }
-
- public String getElementPath() {
- return elementPath;
+ public static VXQueryIndexingDataSource create(int id, String collection, Object type, String function) {
+ return new VXQueryIndexingDataSource(id, collection, new Object[] { type }, function);
}
public String getFunctionCall() {
@@ -71,7 +63,7 @@ public class VXQueryIndexingDataSource extends AbstractVXQueryDataSource {
@Override
public String toString() {
- return "VXQueryIndexingDataSource [collectionName=" + collectionName + ", elementPath=" + elementPath
+ return "VXQueryIndexingDataSource [collectionName=" + collectionName + ", elementPath=" + this.childSeq
+ ", function=" + function + "]";
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
index ac92a0e..3563713 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
@@ -16,15 +16,13 @@
*/
package org.apache.vxquery.metadata;
-import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Map;
+import java.util.List;
import java.util.logging.Logger;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.api.client.NodeControllerInfo;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.IFrameFieldAppender;
import org.apache.hyracks.api.comm.VSizeFrame;
@@ -35,23 +33,20 @@ import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
import org.apache.vxquery.datamodel.values.XDMConstants;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.functions.BuiltinFunctions;
import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
import org.apache.vxquery.runtime.functions.index.VXQueryIndexReader;
-import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
-import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
+import org.apache.vxquery.runtime.functions.index.centralizer.IndexCentralizerUtil;
+import org.apache.vxquery.runtime.functions.index.update.IndexUpdater;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
@@ -61,18 +56,18 @@ public class VXQueryIndexingOperatorDescriptor extends AbstractSingleActivityOpe
private short dataSourceId;
private short totalDataSources;
private String[] collectionPartitions;
- private String elementPath;
private final String functionCall;
+ private List<Integer> childSeq;
public VXQueryIndexingOperatorDescriptor(IOperatorDescriptorRegistry spec, VXQueryIndexingDataSource ds,
- RecordDescriptor rDesc, String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos) {
+ RecordDescriptor rDesc) {
super(spec, 1, 1);
this.functionCall = ds.getFunctionCall();
collectionPartitions = ds.getPartitions();
dataSourceId = (short) ds.getDataSourceId();
totalDataSources = (short) ds.getTotalDataSources();
recordDescriptors[0] = rDesc;
- this.elementPath = ds.getElementPath();
+ childSeq = ds.getChildSeq();
}
@Override
@@ -87,10 +82,11 @@ public class VXQueryIndexingOperatorDescriptor extends AbstractSingleActivityOpe
final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources);
final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
final String collectionName = collectionPartitions[partition % collectionPartitions.length];
- String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
+ final String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil(
ctx.getIOManager().getIODevices().get(0).getMount());
indexCentralizerUtil.readIndexDirectory();
+ final IPointable result = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
@Override
@@ -103,100 +99,115 @@ public class VXQueryIndexingOperatorDescriptor extends AbstractSingleActivityOpe
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
fta.reset(buffer);
- IPointable result = new TaggedValuePointable();
-
- final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
- final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY
- .createPointable();
-
- final ByteBufferInputStream bbis = new ByteBufferInputStream();
- final DataInputStream di = new DataInputStream(bbis);
- final SequenceBuilder sb = new SequenceBuilder();
final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
- String indexModifiedName;
+ abvs.reset();
+ abvsFileNode.reset();
+
if (collectionModifiedName.contains("hdfs://")) {
throw new HyracksDataException("Indexing support for HDFS not yet implemented.");
} else {
if (functionCall.equals(
BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier().getName())) {
- indexModifiedName = indexCentralizerUtil.putIndexForCollection(collectionModifiedName);
- File collectionDirectory = new File(collectionModifiedName);
-
- //check if directory is in the local file system
- if (collectionDirectory.exists() && collectionDirectory.isDirectory()) {
- IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
- try {
- indexConstructorUtil.evaluate(collectionModifiedName, indexModifiedName, result,
- stringp, bbis, di, sb, abvs, nodeIdProvider, abvsFileNode, nodep, false,
- nodeId);
- XDMConstants.setTrue(result);
- FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
- result.getStartOffset(), result.getLength());
- } catch (SystemException e) {
- throw new HyracksDataException("Could not create index for collection: "
- + collectionName + " in dir: " + indexModifiedName + " " + e.getMessage());
- }
- } else {
- throw new HyracksDataException("Cannot find Collection Directory (" + nodeId + ":"
- + collectionDirectory.getAbsolutePath() + ")");
- }
- } else if (functionCall
- .equals(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier().getName())) {
- indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
- IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs,
- nodeIdProvider, abvsFileNode, nodep, nodeId);
try {
- updater.setup();
- updater.updateIndex();
- updater.updateMetadataFile();
- updater.exit();
- XDMConstants.setTrue(result);
- FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
- result.getStartOffset(), result.getLength());
+ createIndex(result, abvs, abvsFileNode);
} catch (IOException e) {
- throw new HyracksDataException(
- "Could not update index in " + indexModifiedName + " " + e.getMessage());
+ throw new HyracksDataException(e);
}
} else if (functionCall
+ .equals(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier().getName())) {
+ updateIndex(result, abvs, abvsFileNode);
+ } else if (functionCall
.equals(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier().getName())) {
- indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
- IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs,
- nodeIdProvider, abvsFileNode, nodep, nodeId);
- indexCentralizerUtil.deleteEntryForCollection(collectionModifiedName);
- try {
- updater.setup();
- updater.deleteAllIndexes();
- XDMConstants.setTrue(result);
- FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
- result.getStartOffset(), result.getLength());
- } catch (IOException e) {
- throw new HyracksDataException(
- "Could not delete index in " + indexModifiedName + " " + e.getMessage());
- }
-
+ deleteIndex(result, abvs, abvsFileNode);
} else if (functionCall
- .equals(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier().getName())) {
- indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
- VXQueryIndexReader indexReader = new VXQueryIndexReader(ctx, indexModifiedName, elementPath);
- try {
- indexReader.init();
- while (indexReader.step(result)) {
- FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
- result.getStartOffset(), result.getLength());
- }
- } catch (AlgebricksException e) {
- throw new HyracksDataException("Could not read index.");
- }
-
+ .equals(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_1.getFunctionIdentifier().getName())
+ || functionCall
+ .equals(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier().getName())) {
+ usingIndex(result);
} else {
throw new HyracksDataException("Unsupported function call (" + functionCall + ")");
}
}
}
+ public void createIndex(IPointable result, ArrayBackedValueStorage abvs,
+ ArrayBackedValueStorage abvsFileNode) throws IOException {
+ String indexModifiedName = indexCentralizerUtil.putIndexForCollection(collectionModifiedName);
+ File collectionDirectory = new File(collectionModifiedName);
+
+ //check if directory is in the local file system
+ if (collectionDirectory.exists() && collectionDirectory.isDirectory()) {
+ IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
+ try {
+ indexConstructorUtil.evaluate(collectionModifiedName, indexModifiedName, result, abvs,
+ nodeIdProvider, abvsFileNode, false, nodeId);
+ XDMConstants.setTrue(result);
+ FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(),
+ result.getLength());
+ } catch (SystemException e) {
+ throw new HyracksDataException("Could not create index for collection: " + collectionName
+ + " in dir: " + indexModifiedName + " " + e.getMessage(), e);
+ }
+ } else {
+ throw new HyracksDataException("Cannot find Collection Directory (" + nodeId + ":"
+ + collectionDirectory.getAbsolutePath() + ")");
+ }
+ }
+
+ public void updateIndex(IPointable result, ArrayBackedValueStorage abvs,
+ ArrayBackedValueStorage abvsFileNode) throws HyracksDataException {
+ String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+ IndexUpdater updater = new IndexUpdater(indexModifiedName, result, abvs, nodeIdProvider, abvsFileNode,
+ nodeId);
+ try {
+ updater.setup();
+ updater.updateIndex();
+ updater.updateMetadataFile();
+ updater.exit();
+ XDMConstants.setTrue(result);
+ FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(),
+ result.getLength());
+ } catch (IOException e) {
+ throw new HyracksDataException(
+ "Could not update index in " + indexModifiedName + " " + e.getMessage(), e);
+ }
+ }
+
+ public void deleteIndex(IPointable result, ArrayBackedValueStorage abvs,
+ ArrayBackedValueStorage abvsFileNode) throws HyracksDataException {
+ String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+ IndexUpdater updater = new IndexUpdater(indexModifiedName, result, abvs, nodeIdProvider, abvsFileNode,
+ nodeId);
+ indexCentralizerUtil.deleteEntryForCollection(collectionModifiedName);
+ try {
+ updater.setup();
+ updater.deleteAllIndexes();
+ XDMConstants.setTrue(result);
+ FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(),
+ result.getLength());
+ } catch (IOException e) {
+ throw new HyracksDataException(
+ "Could not delete index in " + indexModifiedName + " " + e.getMessage(), e);
+ }
+ }
+
+ public void usingIndex(IPointable result) throws HyracksDataException {
+ String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+ VXQueryIndexReader indexReader = new VXQueryIndexReader(ctx, indexModifiedName, childSeq, appender);
+ try {
+ indexReader.init();
+ for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) {
+ while (indexReader.step(result, writer, tupleIndex)) {
+ }
+ }
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException("Could not read index.", e);
+ }
+ }
+
@Override
public void fail() throws HyracksDataException {
writer.fail();
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
index f6644d6..5bb9d1a 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
@@ -59,15 +59,17 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
private final Map<String, File> sourceFileMap;
private final StaticContext staticCtx;
private final String hdfsConf;
+ private final List<String> collections;
private final Map<String, NodeControllerInfo> nodeControllerInfos;
public VXQueryMetadataProvider(String[] nodeList, Map<String, File> sourceFileMap, StaticContext staticCtx,
- String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos) {
+ String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos, List<String> collections) {
this.nodeList = nodeList;
this.sourceFileMap = sourceFileMap;
this.staticCtx = staticCtx;
this.hdfsConf = hdfsConf;
this.nodeControllerInfos = nodeControllerInfos;
+ this.collections = collections;
}
@Override
@@ -111,8 +113,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
constraint = getClusterLocations(nodeList, ds.getPartitionCount());
} else {
rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]);
- scanner = new VXQueryIndexingOperatorDescriptor(jobSpec, (VXQueryIndexingDataSource) ds, rDesc,
- this.hdfsConf, this.nodeControllerInfos);
+ scanner = new VXQueryIndexingOperatorDescriptor(jobSpec, (VXQueryIndexingDataSource) ds, rDesc);
constraint = getClusterLocations(nodeList, ds.getPartitionCount());
}
@@ -142,7 +143,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
@Override
public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc)
- throws AlgebricksException {
+ throws AlgebricksException {
throw new UnsupportedOperationException();
}
@@ -168,7 +169,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys,
List<LogicalVariable> secondaryKeys, List<LogicalVariable> additionalNonKeyFields,
ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
- throws AlgebricksException {
+ throws AlgebricksException {
throw new UnsupportedOperationException();
}
@@ -234,7 +235,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
JobGenContext context, JobSpecification spec, boolean bulkload) throws AlgebricksException {
throw new UnsupportedOperationException();
}
-
+
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<String> dataSource,
IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
@@ -243,7 +244,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
JobSpecification jobSpec, boolean bulkload) throws AlgebricksException {
throw new UnsupportedOperationException();
}
-
+
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getUpsertRuntime(IDataSource<String> dataSource,
IOperatorSchema inputSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
@@ -252,7 +253,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
JobSpecification jobSpec) throws AlgebricksException {
throw new UnsupportedOperationException();
}
-
+
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexUpsertRuntime(
IDataSourceIndex<String, String> dataSourceIndex, IOperatorSchema propagatedSchema,
@@ -263,11 +264,26 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
JobSpecification spec) throws AlgebricksException {
throw new UnsupportedOperationException();
}
-
+
@Override
public Map<String, String> getConfig() {
return new HashMap<>();
}
+ public List<String> getIndexCollections() {
+ return collections;
+
+ }
+
+ public boolean hasIndex(ArrayList<String> collections) {
+ boolean indexExists = false;
+ for (String collection : collections) {
+ indexExists = getIndexCollections().contains(collection);
+ if (!indexExists) {
+ break;
+ }
+ }
+ return indexExists;
+ }
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
index 4706496..2c25752 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
@@ -16,10 +16,14 @@
*/
package org.apache.vxquery.runtime.functions.index;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.concurrent.ConcurrentHashMap;
+
import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -31,38 +35,31 @@ import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.index.IndexDocumentBuilder;
-import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil;
-import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
+import org.apache.vxquery.runtime.functions.index.update.MetaFileUtil;
+import org.apache.vxquery.runtime.functions.index.update.XmlMetadata;
import org.apache.vxquery.runtime.functions.util.FunctionHelper;
import org.apache.vxquery.xmlparser.IParser;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
import org.apache.vxquery.xmlparser.XMLParser;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.text.SimpleDateFormat;
-import java.util.concurrent.ConcurrentHashMap;
-
public class IndexConstructorUtil {
- boolean isMetaFilePresent = false;
- MetaFileUtil metaFileUtil;
- ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
+ private final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
+ private final SequenceBuilder sb = new SequenceBuilder();
+ private boolean isMetaFilePresent = false;
+ private MetaFileUtil metaFileUtil;
+ private ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
- public void evaluate(String collectioFolder, String indexFolder, IPointable result, UTF8StringPointable
- stringp, ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
- ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep,
- boolean isElementPath, String nodeId) throws SystemException {
+ public void evaluate(String collectioFolder, String indexFolder, IPointable result, ArrayBackedValueStorage abvs,
+ ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, boolean isElementPath,
+ String nodeId) throws IOException {
- metaFileUtil = new MetaFileUtil(indexFolder);
-// metaFileUtil = .create(indexFolder);
- isMetaFilePresent = metaFileUtil.isMetaFilePresent();
- metaFileUtil.setCollection(collectioFolder);
+ metaFileUtil = new MetaFileUtil(indexFolder);
+ isMetaFilePresent = metaFileUtil.isMetaFilePresent();
+ metaFileUtil.setCollection(collectioFolder);
File collectionDirectory = new File(collectioFolder);
if (!collectionDirectory.exists()) {
- throw new RuntimeException("The collection directory (" + collectioFolder + ") does not exist.");
+ throw new IOException("The collection directory (" + collectioFolder + ") does not exist.");
}
try {
@@ -80,8 +77,7 @@ public class IndexConstructorUtil {
IndexWriter writer = new IndexWriter(dir, iwc);
//Add files to index
- indexXmlFiles(collectionDirectory, writer, isElementPath, nodep, abvsFileNode, nodeIdProvider, sb, bbis, di,
- nodeId);
+ indexXmlFiles(collectionDirectory, writer, isElementPath, abvsFileNode, nodeIdProvider, sb, nodeId);
if (!isMetaFilePresent) {
// Write metadata map to a file.
@@ -101,14 +97,13 @@ public class IndexConstructorUtil {
}
}
- /*This function goes recursively one file at a time. First it turns the file into an ABVS document node, then
+ /*
+ * This function goes recursively one file at a time. First it turns the file into an ABVS document node, then
* it indexes that document node.
*/
public void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath,
- TaggedValuePointable nodep, ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider,
- SequenceBuilder sb, ByteBufferInputStream bbis, DataInputStream di, String nodeId)
- throws SystemException, IOException {
-
+ ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, SequenceBuilder sb, String nodeId)
+ throws IOException {
SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy, HH:mm:ss");
for (File file : collectionDirectory.listFiles()) {
@@ -116,8 +111,7 @@ public class IndexConstructorUtil {
if (readableXmlFile(file.getPath())) {
abvsFileNode.reset();
- IndexDocumentBuilder ibuilder = getIndexBuilder(file, writer, nodep, abvsFileNode, nodeIdProvider, bbis,
- di, nodeId);
+ IndexDocumentBuilder ibuilder = getIndexBuilder(file, writer, abvsFileNode, nodeIdProvider, nodeId);
ibuilder.printStart();
if (!isMetaFilePresent) {
@@ -131,22 +125,21 @@ public class IndexConstructorUtil {
} else if (file.isDirectory()) {
// Consider all XML file in sub directories.
- indexXmlFiles(file, writer, isElementPath, nodep, abvsFileNode, nodeIdProvider, sb, bbis, di, nodeId);
+ indexXmlFiles(file, writer, isElementPath, abvsFileNode, nodeIdProvider, sb, nodeId);
}
}
}
public boolean readableXmlFile(String path) {
- return (path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz"));
+ return path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz");
}
- public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep,
- ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, ByteBufferInputStream bbis,
- DataInputStream di, String nodeId) throws IOException {
+ public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, ArrayBackedValueStorage abvsFileNode,
+ ITreeNodeIdProvider nodeIdProvider, String nodeId) throws IOException {
//Get the document node
IParser parser = new XMLParser(false, nodeIdProvider, nodeId);
- FunctionHelper.readInDocFromString(file.getPath(), bbis, di, abvsFileNode, parser);
+ FunctionHelper.readInDocFromString(file.getPath(), abvsFileNode, parser);
nodep.set(abvsFileNode.getByteArray(), abvsFileNode.getStartOffset(), abvsFileNode.getLength());
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
deleted file mode 100644
index 6677bd9..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index;
-
-import java.io.IOException;
-
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
-
-public class ShowIndexScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
- private static final long serialVersionUID = 1L;
-
- public ShowIndexScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- super(args);
- }
-
- @Override
- protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
- throws HyracksDataException {
- final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
- final SequenceBuilder sb = new SequenceBuilder();
-
- return new AbstractTaggedValueArgumentScalarEvaluator(args) {
- @Override
- protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
- try {
- abvs.reset();
- sb.reset(abvs);
- IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil(
- ctx.getIOManager().getIODevices().get(0).getMount());
- indexCentralizerUtil.readIndexDirectory();
- indexCentralizerUtil.getAllCollections(sb);
- sb.finish();
- result.set(abvs);
- } catch (IOException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
-
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java
new file mode 100644
index 0000000..6b18b33
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java
@@ -0,0 +1,67 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index;
+
+import java.io.IOException;
+
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
+import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
+import org.apache.vxquery.runtime.functions.index.centralizer.IndexCentralizerUtil;
+
+public class ShowIndexesScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ public ShowIndexesScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+ super(args);
+ }
+
+ @Override
+ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
+ throws HyracksDataException {
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ final SequenceBuilder sb = new SequenceBuilder();
+
+ return new AbstractTaggedValueArgumentScalarEvaluator(args) {
+ @Override
+ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
+ try {
+ abvs.reset();
+ sb.reset(abvs);
+ IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil(
+ ctx.getIOManager().getIODevices().get(0).getMount());
+ indexCentralizerUtil.readIndexDirectory();
+ indexCentralizerUtil.getAllCollections(sb);
+ sb.finish();
+ result.set(abvs);
+ } catch (IOException e) {
+ throw new SystemException(ErrorCode.SYSE0001, e);
+ }
+
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
index 8750849..cf781ab 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
@@ -16,11 +16,17 @@
*/
package org.apache.vxquery.runtime.functions.index;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.comm.IFrameFieldAppender;
+import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
@@ -32,21 +38,21 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
+import org.apache.vxquery.context.DynamicContext;
import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.index.IndexAttributes;
+import org.apache.vxquery.runtime.functions.util.FunctionHelper;
+import org.apache.vxquery.types.ElementType;
+import org.apache.vxquery.types.NameTest;
+import org.apache.vxquery.types.NodeType;
+import org.apache.vxquery.types.SequenceType;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
import org.apache.vxquery.xmlparser.SAXContentHandler;
import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
public class VXQueryIndexReader {
private ArrayBackedValueStorage nodeAbvs = new ArrayBackedValueStorage();
@@ -55,10 +61,7 @@ public class VXQueryIndexReader {
private int indexLength;
private String elementPath;
private String indexName;
-
- private ByteBufferInputStream bbis = new ByteBufferInputStream();
- private DataInputStream di = new DataInputStream(bbis);
-
+ private List<SequenceType> childSequenceTypes;
private IndexReader reader;
private IndexSearcher searcher;
private QueryParser parser;
@@ -68,14 +71,40 @@ public class VXQueryIndexReader {
private Document doc;
private List<IndexableField> fields;
private IHyracksTaskContext ctx;
+ private String[] childLocalName = null;
+ private IFrameFieldAppender appender;
+ private boolean firstElement;
- public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, String elementPath) {
+ public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, List<Integer> childSeq,
+ IFrameFieldAppender appender) {
this.ctx = context;
this.indexName = indexPath;
- this.elementPath = elementPath;
+ this.appender = appender;
+ final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData();
+ childSequenceTypes = new ArrayList<>();
+ for (int typeCode : childSeq) {
+ childSequenceTypes.add(dCtx.getStaticContext().lookupSequenceType(typeCode));
+ }
+ childLocalName = new String[childSequenceTypes.size()];
+ int index = 0;
+ StringBuilder stb = new StringBuilder();
+ stb.append("/");
+ for (SequenceType sType : childSequenceTypes) {
+ NodeType nodeType = (NodeType) sType.getItemType();
+ ElementType eType = (ElementType) nodeType;
+ NameTest nameTest = eType.getNameTest();
+ childLocalName[index] = FunctionHelper.getStringFromBytes(nameTest.getLocalName());
+
+ stb.append(childLocalName[index]);
+ if (index != childSequenceTypes.size() - 1) {
+ stb.append("/");
+ }
+ ++index;
+ }
+ elementPath = stb.toString();
}
- public boolean step(IPointable result) throws AlgebricksException {
+ public boolean step(IPointable result, IFrameWriter writer, int tupleIndex) throws AlgebricksException {
/*each step will create a tuple for a single xml file
* This is done using the parse function
* checkoverflow is used throughout. This is because memory might not be
@@ -88,6 +117,8 @@ public class VXQueryIndexReader {
//TODO: now we get back the entire document
doc = searcher.doc(hits[indexPlace].doc);
fields = doc.getFields();
+ handler.setupElementWriter(writer, tupleIndex);
+ this.firstElement = true;
parse(nodeAbvs);
} catch (IOException e) {
throw new AlgebricksException(e);
@@ -103,7 +134,7 @@ public class VXQueryIndexReader {
int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
- handler = new SAXContentHandler(false, nodeIdProvider, true);
+ handler = new SAXContentHandler(false, nodeIdProvider, appender, childSequenceTypes);
nodeAbvs.reset();
indexPlace = 0;
@@ -125,7 +156,7 @@ public class VXQueryIndexReader {
String queryString = elementPath.replaceAll("/", ".");
queryString = "item:" + queryString + "*";
- int lastslash = elementPath.lastIndexOf("/");
+ int lastslash = elementPath.lastIndexOf('/');
elementPath = elementPath.substring(0, lastslash) + ":" + elementPath.substring(lastslash + 1);
elementPath = elementPath.replaceAll("/", ".") + ".element";
@@ -135,31 +166,25 @@ public class VXQueryIndexReader {
//TODO: Right now it only returns 1000000 results
results = searcher.search(query, 1000000);
-
} catch (Exception e) {
- throw new SystemException(null);
+ throw new SystemException(null, e);
}
hits = results.scoreDocs;
- System.out.println("found: " + results.totalHits);
indexPlace = 0;
indexLength = hits.length;
-
}
public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException {
try {
- handler.startDocument();
-
for (int i = 0; i < fields.size(); i++) {
String fieldValue = fields.get(i).stringValue();
if (fieldValue.equals(elementPath)) {
+ handler.startDocument();
+ this.firstElement = true;
buildElement(abvsFileNode, i);
}
}
-
- handler.endDocument();
- handler.writeDocument(abvsFileNode);
} catch (Exception e) {
throw new IOException(e);
}
@@ -167,6 +192,7 @@ public class VXQueryIndexReader {
private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) throws SAXException {
int whereIFinish = fieldNum;
+ int firstFinish;
IndexableField field = fields.get(fieldNum);
String contents = field.stringValue();
String uri = "";
@@ -176,18 +202,37 @@ public class VXQueryIndexReader {
String type = contents.substring(lastDot + 1);
String lastBit = contents.substring(firstColon + 1, lastDot);
- if (type.equals("textnode")) {
+ if (this.firstElement) {
+ this.firstElement = false;
+ firstFinish = whereIFinish - this.childSequenceTypes.size() + 1;
+ String firstBit = contents.substring(1, firstColon);
+ List<String> names = new ArrayList<>();
+ List<String> values = new ArrayList<>();
+ List<String> uris = new ArrayList<>();
+ List<String> localNames = new ArrayList<>();
+ List<String> types = new ArrayList<>();
+ List<String> qNames = new ArrayList<>();
+ firstFinish = findAttributeChildren(firstFinish, names, values, uris, localNames, types, qNames);
+ Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
+
+ handler.startElement(uri, firstBit, firstBit, atts);
+ buildElement(abvsFileNode, firstFinish + 1);
+ handler.endElement(uri, firstBit, firstBit);
+
+ }
+
+ if ("textnode".equals(type)) {
char[] charContents = lastBit.toCharArray();
handler.characters(charContents, 0, charContents.length);
}
- if (type.equals("element")) {
- List<String> names = new ArrayList<String>();
- List<String> values = new ArrayList<String>();
- List<String> uris = new ArrayList<String>();
- List<String> localNames = new ArrayList<String>();
- List<String> types = new ArrayList<String>();
- List<String> qNames = new ArrayList<String>();
+ if ("element".equals(type)) {
+ List<String> names = new ArrayList<>();
+ List<String> values = new ArrayList<>();
+ List<String> uris = new ArrayList<>();
+ List<String> localNames = new ArrayList<>();
+ List<String> types = new ArrayList<>();
+ List<String> qNames = new ArrayList<>();
whereIFinish = findAttributeChildren(whereIFinish, names, values, uris, localNames, types, qNames);
Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
@@ -264,7 +309,7 @@ public class VXQueryIndexReader {
String adultPath = adultId.substring(0, lastDotAdult);
adultPath = adultPath.replaceFirst(":", ".");
- return (childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + "."));
+ return childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + ".");
}
boolean isDirectChildAttribute(IndexableField child, IndexableField adult) {
@@ -278,7 +323,7 @@ public class VXQueryIndexReader {
String childType = childSegments[childSegments.length - 1];
- return (childPath.startsWith(adultPath + ":") && childType.equals("attribute"));
+ return childPath.startsWith(adultPath + ":") && "attribute".equals(childType);
}
}