You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by sj...@apache.org on 2016/09/07 23:22:45 UTC
[3/4] vxquery git commit: VXQUERY-207 VXQUERY-209 Parallel Index
creation and access, and index centralization
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
index 126ddf5..2f22ade 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
@@ -28,7 +28,6 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.ValueTag;
import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.index.IndexDocumentBuilder;
@@ -39,58 +38,31 @@ import org.apache.vxquery.xmlparser.IParser;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
import org.apache.vxquery.xmlparser.XMLParser;
-import javax.xml.bind.JAXBException;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.nio.file.Paths;
-import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
-import java.util.Arrays;
import java.util.concurrent.ConcurrentHashMap;
public class IndexConstructorUtil {
- static boolean isMetaFilePresent = false;
- static MetaFileUtil metaFileUtil;
- static ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
+ boolean isMetaFilePresent = false;
+ MetaFileUtil metaFileUtil;
+ ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
- public static void evaluate(TaggedValuePointable[] args, IPointable result, UTF8StringPointable stringp,
- ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
+ public void evaluate(String collectioFolder, String indexFolder, IPointable result, UTF8StringPointable
+ stringp, ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep,
- boolean isElementPath, String nodeId) throws SystemException, JAXBException {
- String collectionFolder;
- String indexFolder;
- TaggedValuePointable collectionTVP = args[0];
- TaggedValuePointable indexTVP = args[1];
-
- if (collectionTVP.getTag() != ValueTag.XS_STRING_TAG || indexTVP.getTag() != ValueTag.XS_STRING_TAG) {
- throw new SystemException(ErrorCode.FORG0006);
- }
+ boolean isElementPath, String nodeId) throws SystemException {
- try {
- // Get the list of files.
- collectionTVP.getValue(stringp);
- bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(),
- stringp.getLength() + stringp.getStartOffset())), 0);
- collectionFolder = di.readUTF();
-
- // Get the index folder
- indexTVP.getValue(stringp);
- bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(),
- stringp.getLength() + stringp.getStartOffset())), 0);
- indexFolder = di.readUTF();
-
- metaFileUtil = MetaFileUtil.create(indexFolder);
+ metaFileUtil = new MetaFileUtil(indexFolder);
+// metaFileUtil = .create(indexFolder);
isMetaFilePresent = metaFileUtil.isMetaFilePresent();
- metaFileUtil.setCollectionForIndex(indexFolder, collectionFolder);
+ metaFileUtil.setCollection(collectioFolder);
- } catch (IOException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
- File collectionDirectory = new File(collectionFolder);
+ File collectionDirectory = new File(collectioFolder);
if (!collectionDirectory.exists()) {
- throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist.");
+ throw new RuntimeException("The collection directory (" + collectioFolder + ") does not exist.");
}
try {
@@ -132,7 +104,7 @@ public class IndexConstructorUtil {
/*This function goes recursively one file at a time. First it turns the file into an ABVS document node, then
* it indexes that document node.
*/
- public static void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath,
+ public void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath,
TaggedValuePointable nodep, ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider,
SequenceBuilder sb, ByteBufferInputStream bbis, DataInputStream di, String nodeId)
throws SystemException, IOException {
@@ -153,11 +125,7 @@ public class IndexConstructorUtil {
xmlMetadata.setPath(file.getCanonicalPath());
xmlMetadata.setFileName(file.getName());
xmlMetadata.setLastModified(sdf.format(file.lastModified()));
- try {
- xmlMetadata.setMd5(metaFileUtil.generateMD5(file));
- } catch (NoSuchAlgorithmException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
+ xmlMetadata.setMd5(metaFileUtil.generateMD5(file));
metadataMap.put(file.getCanonicalPath(), xmlMetadata);
}
@@ -168,14 +136,14 @@ public class IndexConstructorUtil {
}
}
- public static boolean readableXmlFile(String path) {
+ public boolean readableXmlFile(String path) {
return (path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz"));
}
/**
* Separated from create index method so that it could be used as a helper function in IndexUpdater
*/
- public static IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep,
+ public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep,
ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, ByteBufferInputStream bbis,
DataInputStream di, String nodeId) throws IOException {
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java
deleted file mode 100644
index e713b20..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.XDMConstants;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
-
-import javax.xml.bind.JAXBException;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-
-/**
- * Delete the index of a given index directory
- */
-public class IndexDeleteEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
- public IndexDeleteEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- super(args);
- }
-
- @Override
- protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
- throws AlgebricksException {
- final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
- final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
- final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
- final ByteBufferInputStream bbis = new ByteBufferInputStream();
- final DataInputStream di = new DataInputStream(bbis);
- final SequenceBuilder sb = new SequenceBuilder();
- final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
- final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
- final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
- final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
-
- return new AbstractTaggedValueArgumentScalarEvaluator(args) {
-
- @Override
- protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
- IndexUpdater updater = new IndexUpdater(args, result, stringp, bbis, di, sb, abvs, nodeIdProvider,
- abvsFileNode, nodep, nodeId);
- try {
- updater.setup();
- updater.deleteAllIndexes();
- XDMConstants.setTrue(result);
- } catch (IOException | NoSuchAlgorithmException | JAXBException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
-
- }
-
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java
deleted file mode 100644
index 3fbc279..0000000
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.XDMConstants;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
-
-import javax.xml.bind.JAXBException;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-
-/**
- * Update the index of collection
- */
-public class IndexUpdaterEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
- public IndexUpdaterEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- super(args);
- }
-
- @Override
- protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
- throws AlgebricksException {
- final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
- final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
- final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
- final ByteBufferInputStream bbis = new ByteBufferInputStream();
- final DataInputStream di = new DataInputStream(bbis);
- final SequenceBuilder sb = new SequenceBuilder();
- final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
- final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
- final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
- final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
-
- return new AbstractTaggedValueArgumentScalarEvaluator(args) {
-
- @Override
- protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
- IndexUpdater updater = new IndexUpdater(args, result, stringp, bbis, di, sb, abvs, nodeIdProvider,
- abvsFileNode, nodep, nodeId);
- try {
- updater.setup();
- updater.updateIndex();
- updater.updateMetadataFile();
- updater.exit();
- XDMConstants.setTrue(result);
- } catch (IOException | NoSuchAlgorithmException | JAXBException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
- }
-
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
new file mode 100644
index 0000000..9b72a34
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
@@ -0,0 +1,65 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index;
+
+import java.io.IOException;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
+import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
+import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
+
+public class ShowIndexScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
+ public ShowIndexScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+ super(args);
+ }
+
+ @Override
+ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
+ throws AlgebricksException {
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ final SequenceBuilder sb = new SequenceBuilder();
+
+ return new AbstractTaggedValueArgumentScalarEvaluator(args) {
+ @Override
+ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
+ try {
+ abvs.reset();
+ sb.reset(abvs);
+ IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil(
+ ctx.getIOManager().getIODevices().get(0).getPath());
+ indexCentralizerUtil.readIndexDirectory();
+ indexCentralizerUtil.getAllCollections(sb);
+ sb.finish();
+ result.set(abvs);
+ } catch (IOException e) {
+ throw new SystemException(ErrorCode.SYSE0001, e);
+ }
+
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
new file mode 100644
index 0000000..8750849
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
@@ -0,0 +1,284 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.index.IndexAttributes;
+import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
+import org.apache.vxquery.xmlparser.SAXContentHandler;
+import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+public class VXQueryIndexReader {
+
+ private ArrayBackedValueStorage nodeAbvs = new ArrayBackedValueStorage();
+
+ private int indexPlace;
+ private int indexLength;
+ private String elementPath;
+ private String indexName;
+
+ private ByteBufferInputStream bbis = new ByteBufferInputStream();
+ private DataInputStream di = new DataInputStream(bbis);
+
+ private IndexReader reader;
+ private IndexSearcher searcher;
+ private QueryParser parser;
+ private ScoreDoc[] hits;
+ private SAXContentHandler handler;
+ private Query query;
+ private Document doc;
+ private List<IndexableField> fields;
+ private IHyracksTaskContext ctx;
+
+ public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, String elementPath) {
+ this.ctx = context;
+ this.indexName = indexPath;
+ this.elementPath = elementPath;
+ }
+
+ public boolean step(IPointable result) throws AlgebricksException {
+ /*each step will create a tuple for a single xml file
+ * This is done using the parse function
+ * checkoverflow is used throughout. This is because memory might not be
+ * able to hold all of the results at once, so we return 1 million at
+ * a time and check when we need to get more
+ */
+ if (indexPlace < indexLength) {
+ nodeAbvs.reset();
+ try {
+ //TODO: now we get back the entire document
+ doc = searcher.doc(hits[indexPlace].doc);
+ fields = doc.getFields();
+ parse(nodeAbvs);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ indexPlace += 1;
+ result.set(nodeAbvs.getByteArray(), nodeAbvs.getStartOffset(), nodeAbvs.getLength());
+ return true;
+ }
+ return false;
+ }
+
+ public void init() throws SystemException {
+
+ int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
+ ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
+ handler = new SAXContentHandler(false, nodeIdProvider, true);
+
+ nodeAbvs.reset();
+ indexPlace = 0;
+
+ try {
+ indexPlace = 0;
+
+ //Create the index reader.
+ reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexName)));
+ } catch (IOException e) {
+ throw new SystemException(ErrorCode.SYSE0001, e);
+ }
+
+ searcher = new IndexSearcher(reader);
+ Analyzer analyzer = new CaseSensitiveAnalyzer();
+
+ parser = new CaseSensitiveQueryParser("item", analyzer);
+
+ String queryString = elementPath.replaceAll("/", ".");
+ queryString = "item:" + queryString + "*";
+
+ int lastslash = elementPath.lastIndexOf("/");
+ elementPath = elementPath.substring(0, lastslash) + ":" + elementPath.substring(lastslash + 1);
+ elementPath = elementPath.replaceAll("/", ".") + ".element";
+
+ TopDocs results = null;
+ try {
+ query = parser.parse(queryString);
+
+ //TODO: Right now it only returns 1000000 results
+ results = searcher.search(query, 1000000);
+
+ } catch (Exception e) {
+ throw new SystemException(null);
+ }
+
+ hits = results.scoreDocs;
+ System.out.println("found: " + results.totalHits);
+ indexPlace = 0;
+ indexLength = hits.length;
+
+ }
+
+ public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException {
+ try {
+ handler.startDocument();
+
+ for (int i = 0; i < fields.size(); i++) {
+ String fieldValue = fields.get(i).stringValue();
+ if (fieldValue.equals(elementPath)) {
+ buildElement(abvsFileNode, i);
+ }
+ }
+
+ handler.endDocument();
+ handler.writeDocument(abvsFileNode);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) throws SAXException {
+ int whereIFinish = fieldNum;
+ IndexableField field = fields.get(fieldNum);
+ String contents = field.stringValue();
+ String uri = "";
+
+ int firstColon = contents.indexOf(':');
+ int lastDot = contents.lastIndexOf('.');
+ String type = contents.substring(lastDot + 1);
+ String lastBit = contents.substring(firstColon + 1, lastDot);
+
+ if (type.equals("textnode")) {
+ char[] charContents = lastBit.toCharArray();
+ handler.characters(charContents, 0, charContents.length);
+
+ }
+ if (type.equals("element")) {
+ List<String> names = new ArrayList<String>();
+ List<String> values = new ArrayList<String>();
+ List<String> uris = new ArrayList<String>();
+ List<String> localNames = new ArrayList<String>();
+ List<String> types = new ArrayList<String>();
+ List<String> qNames = new ArrayList<String>();
+ whereIFinish = findAttributeChildren(whereIFinish, names, values, uris, localNames, types, qNames);
+ Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
+
+ handler.startElement(uri, lastBit, lastBit, atts);
+
+ boolean noMoreChildren = false;
+
+ while (whereIFinish + 1 < fields.size() && !noMoreChildren) {
+ if (isChild(fields.get(whereIFinish + 1), field)) {
+ whereIFinish = buildElement(abvsFileNode, whereIFinish + 1);
+ } else {
+ noMoreChildren = true;
+ }
+ }
+
+ handler.endElement(uri, lastBit, lastBit);
+
+ }
+ return whereIFinish;
+ }
+
+ /*This function creates the attribute children for an element node
+ *
+ */
+ int findAttributeChildren(int fieldnum, List<String> n, List<String> v, List<String> u, List<String> l,
+ List<String> t, List<String> q) {
+ int nextindex = fieldnum + 1;
+ boolean foundattributes = false;
+ if (nextindex < fields.size()) {
+ IndexableField nextguy;
+
+ while (nextindex < fields.size()) {
+ nextguy = fields.get(nextindex);
+ String contents = nextguy.stringValue();
+ int firstcolon = contents.indexOf(':');
+ int lastdot = contents.lastIndexOf('.');
+ String lastbit = contents.substring(firstcolon + 1, lastdot);
+
+ if (isDirectChildAttribute(nextguy, fields.get(fieldnum))) {
+ foundattributes = true;
+ n.add(lastbit);
+ IndexableField nextnextguy = fields.get(nextindex + 1);
+ contents = nextnextguy.stringValue();
+ firstcolon = contents.indexOf(':');
+ lastdot = contents.lastIndexOf('.');
+ String nextlastbit = contents.substring(firstcolon + 1, lastdot);
+ v.add(nextlastbit);
+ u.add(lastbit);
+ l.add(lastbit);
+ t.add(lastbit);
+ q.add(lastbit);
+ } else {
+ break;
+ }
+ nextindex += 2;
+ }
+ }
+ if (foundattributes) {
+ return nextindex - 1;
+
+ } else {
+ return fieldnum;
+ }
+ }
+
+ boolean isChild(IndexableField child, IndexableField adult) {
+ String childId = child.stringValue();
+ String adultId = adult.stringValue();
+
+ int lastDotChild = childId.lastIndexOf('.');
+ int lastDotAdult = adultId.lastIndexOf('.');
+
+ String childPath = childId.substring(0, lastDotChild);
+ String adultPath = adultId.substring(0, lastDotAdult);
+ adultPath = adultPath.replaceFirst(":", ".");
+
+ return (childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + "."));
+ }
+
+ boolean isDirectChildAttribute(IndexableField child, IndexableField adult) {
+ String childId = child.stringValue();
+ String adultId = adult.stringValue();
+
+ String childPath = childId.substring(0, childId.lastIndexOf('.'));
+ String adultPath = adultId.substring(0, adultId.lastIndexOf('.'));
+ adultPath = adultPath.replaceFirst(":", ".");
+ String[] childSegments = child.stringValue().split("\\.");
+
+ String childType = childSegments[childSegments.length - 1];
+
+ return (childPath.startsWith(adultPath + ":") && childType.equals("attribute"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
new file mode 100644
index 0000000..38dcc2a
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+
+import java.io.DataOutput;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.vxquery.datamodel.builders.atomic.StringValueBuilder;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.datamodel.values.ValueTag;
+
+/**
+ * Class for maintaining the centralized index information file.
+ * Index centralization procedure.
+ * User can specify the collection directory in VXQuery.java, ncConfig.ioDevices = <index_directory>.
+ * Then all the indexes will be created in that particular directory in sub-folders corresponding to collections.
+ * There will be a single xml file, located in the directory specified in local.xml, which contains all information
+ * about the existing indexes.
+ * This class can be used to read, add, delete, modify the entries and write the file back to the disk.
+ */
+public class IndexCentralizerUtil {
+
+ private final String FILE_NAME = "VXQuery-Index-Directory.xml";
+ private final List<String> collections = new ArrayList<>();
+ private final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil");
+ private File XML_FILE;
+ private String INDEX_LOCATION;
+ private static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>();
+
+ public IndexCentralizerUtil(File index) {
+ this.INDEX_LOCATION = index.getPath();
+ if (!index.exists()) {
+ try {
+ FileUtils.forceMkdir(index);
+ } catch (IOException e) {
+ LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + INDEX_LOCATION + " " + e);
+ }
+ }
+ XML_FILE = new File(index.getPath() + "/" + FILE_NAME);
+ }
+
+ /**
+ * Get the index directory containing index of the given collection
+ *
+ * @param collection : Collection folder
+ * @return Index folder.
+ */
+ public String getIndexForCollection(String collection) {
+ return indexCollectionMap.get(collection).getIndex();
+ }
+
+ /**
+ * Put the index location corresponding to given collection.
+ * Index location is created by using the last 100 characters of collection.
+ *
+ * @param collection : Collection directory
+ */
+ public String putIndexForCollection(String collection) {
+ int length = collection.replaceAll("/", "").length();
+ String index = collection.replaceAll("/", "");
+ index = INDEX_LOCATION + "/" + (length > 100 ? index.substring(length - 100) : index);
+ IndexLocator il = new IndexLocator();
+ il.setCollection(collection);
+ il.setIndex(index);
+ if (indexCollectionMap.get(collection) != null) {
+ return index;
+ }
+ indexCollectionMap.put(collection, il);
+ return index;
+ }
+
+ /**
+ * Remove the entry for given collection directory.
+ *
+ * @param collection : Collection directory
+ */
+ public void deleteEntryForCollection(String collection) {
+ indexCollectionMap.remove(collection);
+ }
+
+ /**
+ * Prints all collections which have an index created.
+ *
+ * @throws IOException
+ */
+ public void getAllCollections(SequenceBuilder sb) throws IOException {
+ for (String s : collections) {
+ StringValueBuilder svb = new StringValueBuilder();
+ ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ DataOutput output = abvs.getDataOutput();
+ output.write(ValueTag.XS_STRING_TAG);
+ svb.write(s, output);
+ sb.addItem(abvs);
+ }
+ }
+
+ /**
+ * Read the collection, index directory file and populate the HashMap.
+ */
+ public void readIndexDirectory() {
+ if (this.XML_FILE.exists()) {
+ try {
+ JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class);
+ Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
+ IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(this.XML_FILE);
+
+ for (IndexLocator il : indexDirectory.getDirectory()) {
+ indexCollectionMap.put(il.getCollection(), il);
+ this.collections.add(il.getCollection());
+ }
+ } catch (JAXBException e) {
+ LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e);
+ }
+ }
+
+ }
+
+ /**
+ * Write back the contents of the HashMap to the file.
+ */
+ public void writeIndexDirectory() {
+ IndexDirectory id = new IndexDirectory();
+ List<IndexLocator> indexLocators = new ArrayList<>(indexCollectionMap.values());
+ id.setDirectory(indexLocators);
+ try {
+ FileOutputStream fileOutputStream = new FileOutputStream(this.XML_FILE);
+ JAXBContext context = JAXBContext.newInstance(IndexDirectory.class);
+ Marshaller jaxbMarshaller = context.createMarshaller();
+ jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+ jaxbMarshaller.marshal(id, fileOutputStream);
+ } catch (JAXBException | FileNotFoundException e) {
+ LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
new file mode 100644
index 0000000..54d9ad9
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+@XmlRootElement(name = "indexes")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class IndexDirectory implements Serializable{
+
+ @XmlElement(name = "index", type = IndexLocator.class)
+ private List<IndexLocator> directory = new ArrayList<>();
+
+ public List<IndexLocator> getDirectory() {
+ return directory;
+ }
+
+
+ public void setDirectory(List<IndexLocator> directory) {
+ this.directory = directory;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
new file mode 100644
index 0000000..1a33c8b
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlRootElement(name = "Entry")
+public class IndexLocator implements Serializable{
+
+ @XmlAttribute
+ private String collection;
+
+ @XmlAttribute
+ private String index;
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public void setCollection(String collection) {
+ this.collection = collection;
+ }
+
+ public String getIndex() {
+ return index;
+ }
+
+ public void setIndex(String index) {
+ this.index = index;
+ }
+}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
index 9aebfb4..2a45747 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
@@ -22,5 +22,4 @@ package org.apache.vxquery.runtime.functions.index.updateIndex;
public class Constants {
public static String FIELD_PATH = "path";
public static String META_FILE_NAME = "vxquery_index.xml";
- public static String COLLECTION_ENTRY = "collection";
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
index 4588282..ba7cd88 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
@@ -29,7 +29,6 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.ValueTag;
import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.index.IndexDocumentBuilder;
@@ -37,16 +36,12 @@ import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer;
import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import javax.xml.bind.JAXBException;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -57,9 +52,7 @@ import java.util.concurrent.ConcurrentHashMap;
public class IndexUpdater {
private MetaFileUtil metaFileUtil;
private ConcurrentHashMap<String, XmlMetadata> metadataMap;
- private TaggedValuePointable[] args;
private IPointable result;
- private UTF8StringPointable stringp;
private ByteBufferInputStream bbis;
private DataInputStream di;
private SequenceBuilder sb;
@@ -71,19 +64,17 @@ public class IndexUpdater {
private IndexWriter indexWriter;
private Set<String> pathsFromFileList;
private String collectionFolder;
- private XmlMetadata collectionMetadata;
private String indexFolder;
private Logger LOGGER = Logger.getLogger("Index Updater");
private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
+ private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
- //TODO : Implement for paralleizing
- public IndexUpdater(TaggedValuePointable[] args, IPointable result, UTF8StringPointable stringp,
+ public IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp,
ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep,
String nodeId) {
- this.args = args;
+ this.indexFolder = indexFolder;
this.result = result;
- this.stringp = stringp;
this.bbis = bbis;
this.di = di;
this.sb = sb;
@@ -100,35 +91,17 @@ public class IndexUpdater {
*
* @throws SystemException
* @throws IOException
- * @throws NoSuchAlgorithmException
*/
- public void setup() throws SystemException, IOException, NoSuchAlgorithmException, JAXBException {
+ public void setup() throws SystemException, IOException {
- TaggedValuePointable indexTVP = args[0];
+ // Read the metadata file and load the metadata map into memory.
+ metaFileUtil = new MetaFileUtil(indexFolder);
+ metaFileUtil.readMetadataFile();
+ metadataMap = metaFileUtil.getMetadata();
- if (indexTVP.getTag() != ValueTag.XS_STRING_TAG) {
- throw new SystemException(ErrorCode.FORG0006);
- }
-
- try {
- // Get the index folder
- indexTVP.getValue(stringp);
- bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(),
- stringp.getLength() + stringp.getStartOffset())), 0);
- indexFolder = di.readUTF();
-
- // Read the metadata file and load the metadata map into memory.
- metaFileUtil = MetaFileUtil.create(indexFolder);
- metaFileUtil.readMetadataFile();
- metadataMap = metaFileUtil.getMetadata(indexFolder);
-
- // Retrieve the collection folder path.
- // Remove the entry for ease of the next steps.
- collectionFolder = metaFileUtil.getCollection(indexFolder);
-
- } catch (IOException | ClassNotFoundException e) {
- throw new SystemException(ErrorCode.SYSE0001, e);
- }
+ // Retrieve the collection folder path.
+ // Remove the entry for ease of the next steps.
+ collectionFolder = metaFileUtil.getCollection();
abvs.reset();
sb.reset(abvs);
@@ -142,9 +115,8 @@ public class IndexUpdater {
* Wrapper for update index function.
*
* @throws IOException
- * @throws NoSuchAlgorithmException
*/
- public void updateIndex() throws IOException, NoSuchAlgorithmException {
+ public void updateIndex() throws IOException {
File collectionDirectory = new File(collectionFolder);
if (!collectionDirectory.exists()) {
throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist.");
@@ -155,6 +127,7 @@ public class IndexUpdater {
//Detect deleted files and execute the delete index process.
deleteIndexOfDeletedFiles(metadataMap.keySet(), pathsFromFileList);
+ updateMetadataFile();
}
/**
@@ -176,7 +149,7 @@ public class IndexUpdater {
*
* @throws IOException
*/
- public synchronized void updateMetadataFile() throws IOException, JAXBException {
+ public synchronized void updateMetadataFile() throws IOException {
//Write the updated metadata to the file.
metaFileUtil.updateMetadataMap(metadataMap, indexFolder);
metaFileUtil.writeMetadataToFile();
@@ -188,14 +161,14 @@ public class IndexUpdater {
*
* @param collection : Collection folder path
*/
- private void updateIndex(File collection) throws IOException, NoSuchAlgorithmException {
+ private void updateIndex(File collection) throws IOException {
File[] list = collection.listFiles();
assert list != null;
for (File file : list) {
pathsFromFileList.add(file.getCanonicalPath());
- if (IndexConstructorUtil.readableXmlFile(file.getCanonicalPath())) {
+ if (indexConstructorUtil.readableXmlFile(file.getCanonicalPath())) {
XmlMetadata data = metadataMap.get(file.getCanonicalPath());
String md5 = metaFileUtil.generateMD5(file);
@@ -212,7 +185,7 @@ public class IndexUpdater {
//Update index corresponding to the xml file.
indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath()));
- indexDocumentBuilder = IndexConstructorUtil
+ indexDocumentBuilder = indexConstructorUtil
.getIndexBuilder(file, indexWriter, nodep, abvsFileNode, nodeIdProvider, bbis, di,
nodeId);
indexDocumentBuilder.printStart();
@@ -230,7 +203,7 @@ public class IndexUpdater {
// In this case, the xml file has not added to the index. (It is a newly added file)
// Therefore generate a new index for this file and add it to the existing index.
- indexDocumentBuilder = IndexConstructorUtil
+ indexDocumentBuilder = indexConstructorUtil
.getIndexBuilder(file, indexWriter, nodep, abvsFileNode, nodeIdProvider, bbis, di, nodeId);
indexDocumentBuilder.printStart();
@@ -254,15 +227,14 @@ public class IndexUpdater {
* @param metadata : Existing metadata object
* @return : XML metadata object with updated fields.
* @throws IOException
- * @throws NoSuchAlgorithmException
*/
- private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException, NoSuchAlgorithmException {
+ private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException {
- if (metadata == null)
+ if (metadata == null) {
metadata = new XmlMetadata();
-
- metadata.setPath(file.getCanonicalPath());
+ }
metadata.setFileName(file.getName());
+ metadata.setPath(file.getCanonicalPath());
metadata.setMd5(metaFileUtil.generateMD5(file));
metadata.setLastModified(sdf.format(file.lastModified()));
return metadata;
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
index 53b02df..dd099b5 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
@@ -19,7 +19,11 @@ package org.apache.vxquery.runtime.functions.index.updateIndex;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import javax.xml.bind.*;
+import javax.xml.bind.DatatypeConverter;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -40,17 +44,14 @@ public class MetaFileUtil {
private File metaFile;
private Logger LOGGER = Logger.getLogger("MetadataFileUtil");
- private Map<String, ConcurrentHashMap<String, XmlMetadata>> indexes = new ConcurrentHashMap<>();
- private Map<String, String> indexToCollection = new ConcurrentHashMap<>();
+ private String index;
+ private String collection;
+ private ConcurrentHashMap<String, XmlMetadata> indexMap = new ConcurrentHashMap<>();
- private MetaFileUtil(String indexFolder) {
+ public MetaFileUtil(String indexFolder) {
this.metaFile = new File(indexFolder + "/" + Constants.META_FILE_NAME);
}
- public static MetaFileUtil create(String indexFolder) {
- return new MetaFileUtil(indexFolder);
- }
-
/**
* Checks for existing metadata file.
*
@@ -66,88 +67,74 @@ public class MetaFileUtil {
* Otherwise insert new.
* @param metadataMap : Set of XmlMetaData objects.
* @param index : The path to index location.
- * @throws IOException
*/
- public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) throws
- IOException, JAXBException {
-
- if (this.indexes.get(index) == null) {
- this.indexes.put(index, metadataMap);
- } else {
- this.indexes.replace(index, metadataMap);
- }
+ public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) {
+ this.indexMap = metadataMap;
+ this.index = index;
}
/**
* Method to get the set of xml metadata for a given collection
*
- * @param index : The collection from which the metadata should be read.
- * @return : Map containing the set of XmlMetadata objects.
- * @throws IOException
- * @throws ClassNotFoundException
+ * @return : Map containing the set of XmlMetadata objects.\
*/
- public ConcurrentHashMap<String, XmlMetadata> getMetadata(String index)
- throws IOException, ClassNotFoundException, JAXBException {
-
- return this.indexes.get(index);
+ public ConcurrentHashMap<String, XmlMetadata> getMetadata() {
+ return this.indexMap;
}
/**
* Read the metadata file and create an in-memory map containing collection paths and xml files.
- * @throws JAXBException
*/
- public void readMetadataFile() throws JAXBException {
- JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
- Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
- VXQueryIndex indexes = (VXQueryIndex) jaxbUnmarshaller.unmarshal(metaFile);
-
- List<XmlMetadataCollection> list = indexes.getIndex();
-
-
- for (XmlMetadataCollection collection : list) {
- String indexPath = collection.getIndexLocation();
- ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
- List<XmlMetadata> metadata = collection.getMetadataList();
+ public void readMetadataFile() {
+ try {
+ JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
+ Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
+ XmlMetadataCollection indexes = (XmlMetadataCollection) jaxbUnmarshaller.unmarshal(metaFile);
- this.indexToCollection.put(indexPath, collection.getCollection());
+ this.collection = indexes.getCollection();
+ this.index = indexes.getIndexLocation();
- for (XmlMetadata mData : metadata) {
- metadataMap.put(mData.getPath(), mData);
+ for (XmlMetadata metadata : indexes.getMetadataList()) {
+ this.indexMap.put(index, metadata);
+ }
+ } catch (JAXBException e) {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e);
}
- this.indexes.put(indexPath, metadataMap);
}
}
/**
* Write the content of the ConcurrentHashMap to the xml metadata file.
- * @throws FileNotFoundException
- * @throws JAXBException
*/
- public void writeMetadataToFile() throws FileNotFoundException, JAXBException {
- VXQueryIndex index = new VXQueryIndex();
- List<XmlMetadataCollection> xmlMetadataCollections = new ArrayList<>();
-
- for (Map.Entry<String, ConcurrentHashMap<String, XmlMetadata>> entry : indexes.entrySet()) {
- XmlMetadataCollection metadataCollection = new XmlMetadataCollection();
- List<XmlMetadata> metadataList = new ArrayList<>();
- metadataCollection.setIndexLocation(entry.getKey());
- metadataCollection.setCollection(indexToCollection.get(entry.getKey()));
- metadataList.addAll(entry.getValue().values());
- metadataCollection.setMetadataList(metadataList);
- xmlMetadataCollections.add(metadataCollection);
- }
- index.setIndex(xmlMetadataCollections);
+ public void writeMetadataToFile() {
+ XmlMetadataCollection collection = new XmlMetadataCollection();
+ List<XmlMetadata> metadataList = new ArrayList<>();
+ for (Map.Entry<String, XmlMetadata> entry : this.indexMap.entrySet()) {
+ metadataList.add(entry.getValue());
+ }
- FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile);
- JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
- Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
- jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
- jaxbMarshaller.marshal(index, fileOutputStream);
+ collection.setMetadataList(metadataList);
+ collection.setCollection(this.collection);
+ collection.setIndexLocation(this.index);
+ try{
+ FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile);
+ JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
+ Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
+ jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+ jaxbMarshaller.marshal(collection, fileOutputStream);
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!");
+ }
+ } catch (JAXBException | FileNotFoundException e) {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e);
+ }
+ }
- if (LOGGER.isDebugEnabled())
- LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!");
}
@@ -157,14 +144,20 @@ public class MetaFileUtil {
*
* @param file : File which the checksum should be generated.
* @return : Checksum String
- * @throws NoSuchAlgorithmException
* @throws IOException
*/
- public String generateMD5(File file) throws NoSuchAlgorithmException, IOException {
- MessageDigest md = MessageDigest.getInstance("MD5");
- md.update(Files.readAllBytes(file.toPath()));
- byte[] md5 = md.digest();
- return DatatypeConverter.printHexBinary(md5);
+ public String generateMD5(File file) throws IOException {
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update(Files.readAllBytes(file.toPath()));
+ byte[] md5 = md.digest();
+ return DatatypeConverter.printHexBinary(md5);
+ } catch (NoSuchAlgorithmException e) {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.log(Level.ERROR, "No Such Algorithm Error " + e.getMessage());
+ }
+ return null;
+ }
}
/**
@@ -189,21 +182,17 @@ public class MetaFileUtil {
/**
* Get the collection for a given index location.
- * @param index : path to index
- * @return
+ * @return collection folder for a given index.
*/
- public String getCollection(String index) {
- return this.indexToCollection.get(index);
+ public String getCollection() {
+ return this.collection;
}
/**
* Set the entry for given index and collection.
- * @param index : path to index
* @param collection : path to corresponding collection
*/
- public void setCollectionForIndex(String index, String collection) {
- if (this.indexToCollection.get(index)==null) {
- this.indexToCollection.put(index, collection);
- }
+ public void setCollection(String collection) {
+ this.collection = collection;
}
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
index 270fb8d..1f5c3e9 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
@@ -16,7 +16,11 @@
*/
package org.apache.vxquery.runtime.functions.index.updateIndex;
-import javax.xml.bind.annotation.*;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
import java.util.List;
/**
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
index 6fa92e1..543719d 100644
--- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
+++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
@@ -14,7 +14,14 @@
*/
package org.apache.vxquery.indexing;
-import junit.framework.Assert;
+import java.io.File;
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.xml.bind.JAXBException;
+
import org.apache.commons.io.FileUtils;
import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil;
import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
@@ -24,12 +31,7 @@ import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
-import javax.xml.bind.JAXBException;
-import java.io.File;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
+import junit.framework.Assert;
/**
* Test cases for testing MetaFileUtil functions.
@@ -49,7 +51,7 @@ public class MetaFileUtilTest {
@BeforeClass
public static void setup() {
new File(TestConstants.INDEX_DIR).mkdir();
- metaFileUtil = MetaFileUtil.create(TestConstants.INDEX_DIR);
+ metaFileUtil = new MetaFileUtil(TestConstants.INDEX_DIR);
initialMap = TestConstants.getInitialMap();
modifiedMap = TestConstants.getModifiedMap();
}
@@ -58,7 +60,7 @@ public class MetaFileUtilTest {
* Test case for generating MD5 string for an XML file.
*/
@Test
- public void step1_testGenerateMD5ForXML() throws IOException, NoSuchAlgorithmException {
+ public void step1_testGenerateMD5ForXML() throws IOException {
TestConstants.createXML("catalog.xml");
File xml = new File(TestConstants.XML_FILE);
String md5 = metaFileUtil.generateMD5(xml);
@@ -71,7 +73,7 @@ public class MetaFileUtilTest {
* Test the creation of metadata file.
*/
@Test
- public void step2_testCreateMetaDataFile() throws IOException, JAXBException {
+ public void step2_testCreateMetaDataFile() {
ConcurrentHashMap<String, XmlMetadata> initialMap = TestConstants.getInitialMap();
metaFileUtil.updateMetadataMap(initialMap, "");
metaFileUtil.writeMetadataToFile();
@@ -82,8 +84,8 @@ public class MetaFileUtilTest {
* Validate the content of the file.
*/
@Test
- public void step3_testValidateMetadataFile() throws IOException, ClassNotFoundException, JAXBException {
- ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata("");
+ public void step3_testValidateMetadataFile() {
+ ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata();
Set<String> from = fromFile.keySet();
Set<String> initial = initialMap.keySet();
@@ -100,7 +102,7 @@ public class MetaFileUtilTest {
* Change the xml file and test whether the changes are detected.
*/
@Test
- public void step4_testDetectFileChanges() throws IOException, NoSuchAlgorithmException {
+ public void step4_testDetectFileChanges() throws IOException {
TestConstants.createXML("catalog_edited.xml");
File xml = new File(TestConstants.XML_FILE);
Assert.assertTrue(metaFileUtil.generateMD5(xml).equals(TestConstants.CHANGED_MD5));
@@ -112,7 +114,7 @@ public class MetaFileUtilTest {
@Test
public void step5_testUpdateMetadata()
throws IOException, ClassNotFoundException, NoSuchAlgorithmException, JAXBException {
- ConcurrentHashMap<String, XmlMetadata> fromFileMap = metaFileUtil.getMetadata("");
+ ConcurrentHashMap<String, XmlMetadata> fromFileMap = metaFileUtil.getMetadata();
XmlMetadata modified = fromFileMap.get(TestConstants.XML_FILE);
File xml = new File(TestConstants.XML_FILE);
@@ -122,7 +124,7 @@ public class MetaFileUtilTest {
metaFileUtil.updateMetadataMap(fromFileMap, TestConstants.INDEX_DIR);
- Assert.assertNotNull(metaFileUtil.getMetadata(TestConstants.INDEX_DIR));
+ Assert.assertNotNull(metaFileUtil.getMetadata());
}
@@ -130,8 +132,8 @@ public class MetaFileUtilTest {
* Validate the updated metadata.
*/
@Test
- public void step6_testVerifyMetadataChange() throws IOException, ClassNotFoundException, JAXBException {
- ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata(TestConstants.INDEX_DIR);
+ public void step6_testVerifyMetadataChange() {
+ ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata();
Set<String> from = fromFile.keySet();
Set<String> modified = modifiedMap.keySet();
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
index 68d40b5..b79107e 100644
--- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
+++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
@@ -16,7 +16,13 @@ package org.apache.vxquery.indexing;
import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.concurrent.ConcurrentHashMap;
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/conf/cluster_example.xml
----------------------------------------------------------------------
diff --git a/vxquery-server/src/main/resources/conf/cluster_example.xml b/vxquery-server/src/main/resources/conf/cluster_example.xml
index 18d9173..f43b22a 100644
--- a/vxquery-server/src/main/resources/conf/cluster_example.xml
+++ b/vxquery-server/src/main/resources/conf/cluster_example.xml
@@ -17,6 +17,7 @@
<cluster xmlns="cluster">
<name>local</name>
<username>joe</username>
+ <index_directory>/tmp/indexFolder</index_directory>
<master_node>
<id>master</id>
<client_ip>128.195.52.177</client_ip>
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/conf/local.xml
----------------------------------------------------------------------
diff --git a/vxquery-server/src/main/resources/conf/local.xml b/vxquery-server/src/main/resources/conf/local.xml
index 4a48620..e0d07f8 100644
--- a/vxquery-server/src/main/resources/conf/local.xml
+++ b/vxquery-server/src/main/resources/conf/local.xml
@@ -16,6 +16,7 @@
-->
<cluster xmlns="cluster">
<name>local</name>
+ <index_directory>/tmp/indexFolder</index_directory>
<master_node>
<id>master</id>
<client_ip>127.0.0.1</client_ip>
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/scripts/cluster_actions.py
----------------------------------------------------------------------
diff --git a/vxquery-server/src/main/resources/scripts/cluster_actions.py b/vxquery-server/src/main/resources/scripts/cluster_actions.py
index deeee33..8ce1607 100644
--- a/vxquery-server/src/main/resources/scripts/cluster_actions.py
+++ b/vxquery-server/src/main/resources/scripts/cluster_actions.py
@@ -120,7 +120,7 @@ class ClusterActions:
def start_nc(self, machine, cc):
print "Start Node Controller."
print " " + machine.get_id() + " " + machine.get_ip()
- command = "./vxquery-server/target/appassembler/bin/startnc.sh " + machine.get_id() + " " + machine.get_ip() + " " + cc.get_client_ip() + " \"" + cc.get_client_port() + "\" \"" + machine.get_java_opts() + "\""
+ command = "./vxquery-server/target/appassembler/bin/startnc.sh " + machine.get_id() + " " + machine.get_ip() + " " + cc.get_client_ip() + " \"" + cc.get_client_port() + "\" \"" + machine.get_java_opts() + "\" \"" + self.ci.get_index_directory() + "\""
self.run_remote_command(machine.get_username(), machine.get_id(), command)
def stop_cc_and_all_ncs(self, machine):
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/scripts/cluster_information.py
----------------------------------------------------------------------
diff --git a/vxquery-server/src/main/resources/scripts/cluster_information.py b/vxquery-server/src/main/resources/scripts/cluster_information.py
index 94b231d..d70828d 100644
--- a/vxquery-server/src/main/resources/scripts/cluster_information.py
+++ b/vxquery-server/src/main/resources/scripts/cluster_information.py
@@ -28,6 +28,9 @@ class ClusterInformation:
def get_java_opts(self):
return get_tag_text(self.config, "java_opts")
+ def get_index_directory(self):
+ return get_tag_text(self.config, "index_directory");
+
def get_master_node_machine(self):
master_node = self.config.getElementsByTagName("master_node")[0]
id = NodeXmlReader.get_cluster_id(master_node)
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/scripts/startnc.sh
----------------------------------------------------------------------
diff --git a/vxquery-server/src/main/resources/scripts/startnc.sh b/vxquery-server/src/main/resources/scripts/startnc.sh
index a199ec8..d719d22 100755
--- a/vxquery-server/src/main/resources/scripts/startnc.sh
+++ b/vxquery-server/src/main/resources/scripts/startnc.sh
@@ -24,6 +24,7 @@ IPADDR=$2
CCHOST=$3
CCPORT=$4
J_OPTS=$5
+IO_DEVICES=$6
#Import cluster properties
MYDIR="$(dirname -- $0)"
@@ -62,6 +63,9 @@ NC_OPTIONS=" -cc-host ${CCHOST} -cluster-net-ip-address ${IPADDR} -data-ip-addr
NC_OPTIONS=" ${NC_OPTIONS} ${NCOPTS} "
+[ "${IO_DEVICES}" ] &&
+ NC_OPTIONS=" ${NC_OPTIONS} -iodevices ${IO_DEVICES} "
+
echo "${JAVA_OPTS}" &> ${NCLOGS_DIR}/nc.log
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
index 75dce1b..d7db8a8 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
@@ -17,16 +17,16 @@
package org.apache.vxquery.xtest;
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.NodeControllerService;
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
public class TestClusterUtil {
private static int clientNetPort = 39000;
@@ -34,6 +34,7 @@ public class TestClusterUtil {
private static int profileDumpPeriod = 10000;
private static String ccHost = "localhost";
private static String nodeId = "nc1";
+ private static String ioDevices = "target/tmp/indexFolder";
private TestClusterUtil() {
}
@@ -58,6 +59,7 @@ public class TestClusterUtil {
ncConfig1.dataIPAddress = publicAddress;
ncConfig1.resultIPAddress = publicAddress;
ncConfig1.nodeId = nodeId;
+ ncConfig1.ioDevices = ioDevices;
return ncConfig1;
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
index c6b339b..78ed8ff 100644
--- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
+++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
@@ -80,6 +80,7 @@ public abstract class AbstractXQueryTest {
case EXPECTED_RESULT_GOT_SAME_RESULT:
break;
case NO_RESULT_FILE:
+ fail(result.state + " (" + result.time + " ms): " + result.testCase.getXQueryDisplayName());
break;
}
}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt
new file mode 100644
index 0000000..f32a580
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt
@@ -0,0 +1 @@
+true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt
new file mode 100644
index 0000000..f32a580
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt
@@ -0,0 +1 @@
+true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt
new file mode 100644
index 0000000..17d6fec
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt
@@ -0,0 +1 @@
+src/test/resources/TestSources/ghcnd
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt
@@ -0,0 +1 @@
+
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt
new file mode 100644
index 0000000..f32a580
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt
@@ -0,0 +1 @@
+true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt
new file mode 100644
index 0000000..baf9dca
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt
new file mode 100644
index 0000000..ef8dde4
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt
@@ -0,0 +1 @@
+<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt
@@ -0,0 +1 @@
+2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt
new file mode 100644
index 0000000..f30101c
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt
@@ -0,0 +1 @@
+3.3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt
new file mode 100644
index 0000000..c84c360
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt
new file mode 100644
index 0000000..9abedff
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt
@@ -0,0 +1,2 @@
+<station><id>GHCND:US000000001</id><displayName>Station 1</displayName><latitude>10.000</latitude><longitude>-10.000</longitude><elevation>1000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 1</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
+<station><id>GHCND:US000000002</id><displayName>Station 2</displayName><latitude>20.000</latitude><longitude>-20.000</longitude><elevation>2000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 2</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt
new file mode 100644
index 0000000..c84c360
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt
new file mode 100644
index 0000000..0be5d98
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt
@@ -0,0 +1,2 @@
+true
+true
\ No newline at end of file