You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2014/02/13 18:17:55 UTC
[05/10] git commit: Allows the xml parser to be reused per partition.
Allows the xml parser to be reused per partition.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/9445d895
Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/9445d895
Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/9445d895
Branch: refs/heads/prestonc/benchmarks
Commit: 9445d89524da703c0cdfe1804228830039cc13d0
Parents: 07a42ea
Author: Preston Carman <pr...@apache.org>
Authored: Wed Feb 12 00:28:40 2014 -0800
Committer: Preston Carman <pr...@apache.org>
Committed: Wed Feb 12 00:28:40 2014 -0800
----------------------------------------------------------------------
.../builders/nodes/DictionaryBuilder.java | 7 +++++
.../VXQueryCollectionOperatorDescriptor.java | 8 +++---
.../runtime/functions/util/FunctionHelper.java | 16 ++++++++++++
.../vxquery/xmlparser/SAXContentHandler.java | 2 ++
.../org/apache/vxquery/xmlparser/XMLParser.java | 27 ++++++++++++++++++++
5 files changed, 57 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/9445d895/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/DictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/DictionaryBuilder.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/DictionaryBuilder.java
index 41cc98c..20b7333 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/DictionaryBuilder.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/DictionaryBuilder.java
@@ -78,6 +78,13 @@ public class DictionaryBuilder {
tempOut = new DataOutputStream(tempStringData);
tempStringPointable = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
}
+
+ public void reset() {
+ stringEndOffsets.clear();
+ sortedSlotIndexes.clear();
+ dataBuffer.reset();
+ tempStringData.reset();
+ }
public void write(ArrayBackedValueStorage abvs) throws IOException {
DataOutput out = abvs.getDataOutput();
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/9445d895/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
index 8e18485..593b8ad 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
@@ -27,11 +27,12 @@ import org.apache.vxquery.datamodel.accessors.PointablePool;
import org.apache.vxquery.datamodel.accessors.PointablePoolFactory;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.runtime.functions.step.ChildPathStep;
+import org.apache.vxquery.runtime.functions.step.ChildPathStepOperatorDescriptor;
import org.apache.vxquery.runtime.functions.util.FunctionHelper;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
import org.apache.vxquery.xmlparser.SAXContentHandler;
import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
+import org.apache.vxquery.xmlparser.XMLParser;
import org.xml.sax.InputSource;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -80,9 +81,10 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO
final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
final int frameSize = ctx.getFrameSize();
final PointablePool ppool = PointablePoolFactory.INSTANCE.createPointablePool();
- final ChildPathStep childPathStep = new ChildPathStep(ctx, ppool);
+ final ChildPathStepOperatorDescriptor childPathStep = new ChildPathStepOperatorDescriptor(ctx, ppool);
final String collectionName = collectionPartitions[partition % collectionPartitions.length];
+ final XMLParser parser = new XMLParser(false, nodeIdProvider);;
return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
@Override
@@ -120,7 +122,7 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO
// Now add new field.
abvsFileNode.reset();
try {
- FunctionHelper.readInDocFromString(file, in, abvsFileNode, nodeIdProvider);
+ FunctionHelper.readInDocFromString(file, in, abvsFileNode, parser);
} catch (Exception e) {
throw new HyracksDataException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/9445d895/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
index e96c8a9..f444bc7 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
@@ -1677,6 +1677,22 @@ public class FunctionHelper {
}
public static void readInDocFromString(File file, InputSource in, ArrayBackedValueStorage abvs,
+ XMLParser parser) throws SystemException {
+ try {
+ if (file.getName().toLowerCase().endsWith(".xml.gz")) {
+ in.setCharacterStream(new InputStreamReader(new GZIPInputStream(new FileInputStream(file))));
+ } else {
+ in.setCharacterStream(new InputStreamReader(new FileInputStream(file)));
+ }
+ parser.parseInputSource(in, abvs);
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException("The file (" + file.getName() + ") does not exist.");
+ } catch (IOException e) {
+ throw new SystemException(ErrorCode.SYSE0001, e);
+ }
+ }
+
+ public static void readInDocFromString(File file, InputSource in, ArrayBackedValueStorage abvs,
ITreeNodeIdProvider treeNodeIdProvider) throws SystemException {
try {
if (file.getName().toLowerCase().endsWith(".xml.gz")) {
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/9445d895/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
index ecf4d7d..a8ec0b9 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
@@ -162,6 +162,8 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
@Override
public void startDocument() throws SAXException {
try {
+ db.reset();
+ docABVS.reset();
docb.reset(docABVS);
if (createNodeIds) {
docb.setLocalNodeId(nodeIdCounter++);
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/9445d895/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
index fee41df..c2ada24 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
@@ -20,9 +20,36 @@ import org.xml.sax.InputSource;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.XMLReaderFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
public class XMLParser {
+ XMLReader parser;
+ SAXContentHandler handler;
+
+ public XMLParser(boolean attachTypes, ITreeNodeIdProvider idProvider) throws HyracksDataException {
+ try {
+ parser = XMLReaderFactory.createXMLReader();
+ handler = new SAXContentHandler(attachTypes, idProvider);
+ parser.setContentHandler(handler);
+ parser.setProperty("http://xml.org/sax/properties/lexical-handler", handler);
+ } catch (Exception e) {
+ throw new HyracksDataException(e.toString());
+ }
+ }
+
+ public void parseInputSource(InputSource in, ArrayBackedValueStorage abvs) throws HyracksDataException {
+ try {
+ parser.parse(in);
+ handler.write(abvs);
+ } catch (Exception e) {
+ throw new HyracksDataException(e.toString());
+ }
+ }
+
+ public void reset() throws SystemException {
+ }
+
public static void parseInputSource(InputSource in, ArrayBackedValueStorage abvs, boolean attachTypes,
ITreeNodeIdProvider idProvider) throws SystemException {
XMLReader parser;