You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2014/05/21 03:42:54 UTC
git commit: More clean up and fixes for new SAXContentHandler
features.
Repository: incubator-vxquery
Updated Branches:
refs/heads/prestonc/benchmark cf09bd9ae -> 8b86884a1
More clean up and fixes for new SAXContentHandler features.
- Made doc and doc-available work with the old SAXContentHandler method. (Basically the element writer does not affect the pervious version.)
- Better variable and function naming.
- Attempt to make it clean to create XMLParser and SAXContentHandler.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/8b86884a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/8b86884a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/8b86884a
Branch: refs/heads/prestonc/benchmark
Commit: 8b86884a11ccf08e6b315fefdfb62184083f3f0d
Parents: cf09bd9
Author: Preston Carman <pr...@apache.org>
Authored: Tue May 20 18:42:34 2014 -0700
Committer: Preston Carman <pr...@apache.org>
Committed: Tue May 20 18:42:34 2014 -0700
----------------------------------------------------------------------
.../VXQueryCollectionOperatorDescriptor.java | 10 +-
.../FnDocAvailableScalarEvaluatorFactory.java | 10 +-
.../runtime/functions/util/FunctionHelper.java | 4 +-
.../vxquery/xmlparser/SAXContentHandler.java | 96 +++++++++++---------
.../org/apache/vxquery/xmlparser/XMLParser.java | 25 ++++-
5 files changed, 88 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/8b86884a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
index 1f4bb2f..a9c7255 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
@@ -71,20 +71,16 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO
final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources);
final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData();
- final List<SequenceType> childSequenceTypes = new ArrayList<SequenceType>();
final String collectionName = collectionPartitions[partition % collectionPartitions.length];
- final XMLParser parser = new XMLParser(false, nodeIdProvider);;
+ final XMLParser parser = new XMLParser(false, nodeIdProvider, frame, appender, childSeq,
+ dCtx.getStaticContext());
return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
@Override
public void open() throws HyracksDataException {
appender.reset(frame, true);
writer.open();
-
- for (int typeCode : childSeq) {
- childSequenceTypes.add(dCtx.getStaticContext().lookupSequenceType(typeCode));
- }
}
@Override
@@ -100,7 +96,7 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO
Iterator<File> it = FileUtils.iterateFiles(collectionDirectory, new VXQueryIOFileFilter(),
TrueFileFilter.INSTANCE);
while (it.hasNext()) {
- parser.parseOutElements(it.next(), frame, appender, writer, fta, t, childSequenceTypes);
+ parser.parseOutElements(it.next(), writer, fta, t);
}
}
} else {
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/8b86884a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
index 499119c..ad4b1f0 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
@@ -27,6 +27,9 @@ import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
import org.apache.vxquery.runtime.functions.util.FunctionHelper;
+import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
+import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
+import org.apache.vxquery.xmlparser.XMLParser;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
@@ -52,6 +55,8 @@ public class FnDocAvailableScalarEvaluatorFactory extends AbstractTaggedValueArg
final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable();
final ByteBufferInputStream bbis = new ByteBufferInputStream();
final DataInputStream di = new DataInputStream(bbis);
+ final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
+ final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
return new AbstractTaggedValueArgumentScalarEvaluator(args) {
@Override
@@ -69,9 +74,10 @@ public class FnDocAvailableScalarEvaluatorFactory extends AbstractTaggedValueArg
if (tvp.getTag() != ValueTag.XS_STRING_TAG) {
throw new SystemException(ErrorCode.FORG0006);
}
-
+ tvp.getValue(stringp);
try {
- FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, null);
+ XMLParser parser = new XMLParser(false, nodeIdProvider);
+ FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser);
XDMConstants.setTrue(result);
} catch (Exception e) {
XDMConstants.setFalse(result);
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/8b86884a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
index 4953c12..8074eab 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
@@ -1232,8 +1232,8 @@ public class FunctionHelper {
System.err.println(" printUTF8String END");
}
- public static void readInDocFromPointable(UTF8StringPointable stringp, ByteBufferInputStream bbis, DataInputStream di,
- ArrayBackedValueStorage abvs, XMLParser parser) throws HyracksDataException {
+ public static void readInDocFromPointable(UTF8StringPointable stringp, ByteBufferInputStream bbis,
+ DataInputStream di, ArrayBackedValueStorage abvs, XMLParser parser) throws HyracksDataException {
String fName;
try {
fName = getStringFromPointable(stringp, bbis, di);
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/8b86884a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
index 58a4f03..bd1a354 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
@@ -96,7 +96,7 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
private final ArrayBackedValueStorage resultABVS;
- private boolean writeMode;
+ private boolean skipping;
private boolean[] subElement = null;
@@ -129,12 +129,20 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
freeENBList = new ArrayList<ElementNodeBuilder>();
pendingText = false;
tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
- writeMode = false;
+ skipping = true;
+ }
+
+ public SAXContentHandler(boolean attachTypes, ITreeNodeIdProvider nodeIdProvider, ByteBuffer frame,
+ FrameTupleAppender appender, List<SequenceType> childSequenceTypes) {
+ this(attachTypes, nodeIdProvider);
+ this.frame = frame;
+ this.appender = appender;
+ setChildPathSteps(childSequenceTypes);
}
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
- if (writeMode) {
+ if (!skipping) {
buffer.append(ch, start, length);
pendingText = true;
}
@@ -142,12 +150,14 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
@Override
public void endDocument() throws SAXException {
- if (writeMode) {
+ if (!skipping) {
try {
flushText();
docb.endChildrenChunk();
docb.finish();
- writeElement();
+ if (frame != null && appender != null) {
+ writeElement();
+ }
} catch (IOException e) {
e.printStackTrace();
throw new SAXException(e);
@@ -155,22 +165,25 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
}
}
+ private void endElementChildPathStep() throws IOException {
+ if (foundFirstNonSkippedElement()) {
+ writeElement();
+ }
+ if (subElement != null && depth <= subElement.length) {
+ subElement[depth - 1] = false;
+ }
+ }
+
@Override
public void endElement(String uri, String localName, String name) throws SAXException {
- if (writeMode) {
+ if (!skipping) {
try {
flushText();
ElementNodeBuilder enb = enbStack.remove(enbStack.size() - 1);
enb.endChildrenChunk();
endChildInParent(enb);
-
- if (foundChildPathStep()) {
- writeElement();
- }
- if (subElement != null && depth <= subElement.length) {
- subElement[depth - 1] = false;
- }
freeENB(enb);
+ endElementChildPathStep();
} catch (IOException e) {
e.printStackTrace();
throw new SAXException(e);
@@ -189,7 +202,7 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
@Override
public void processingInstruction(String target, String data) throws SAXException {
- if (writeMode) {
+ if (!skipping) {
try {
flushText();
startChildInParent(pinb);
@@ -221,43 +234,47 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
@Override
public void startDocument() throws SAXException {
if (subElement == null) {
- writeMode = true;
+ skipping = false;
}
- try {
- db.reset();
- docABVS.reset();
- if (writeMode) {
+ db.reset();
+ docABVS.reset();
+ if (!skipping) {
+ try {
docb.reset(docABVS);
if (createNodeIds) {
docb.setLocalNodeId(nodeIdCounter++);
}
docb.startChildrenChunk();
flushText();
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new SAXException(e);
}
- } catch (IOException e) {
- e.printStackTrace();
- throw new SAXException(e);
}
}
- @Override
- public void startElement(String uri, String localName, String name, Attributes atts) throws SAXException {
- depth++;
- // Check path step if it exists.
+ private boolean startElementChildPathStep(String uri, String localName) {
if (subElement != null && depth <= subElement.length) {
+ // Check path step if it exists.
if (uri.compareTo(childUri[depth - 1]) == 0) {
if (localName.compareTo(childLocalName[depth - 1]) == 0) {
subElement[depth - 1] = true;
}
}
}
-
- boolean start = foundChildPathStep();
+ boolean start = foundFirstNonSkippedElement();
if (start) {
- writeMode = true;
+ skipping = false;
}
+ return start;
+ }
- if (writeMode) {
+ @Override
+ public void startElement(String uri, String localName, String name, Attributes atts) throws SAXException {
+ depth++;
+ boolean start = startElementChildPathStep(uri, localName);
+
+ if (!skipping) {
try {
flushText();
int idx = name.indexOf(':');
@@ -320,7 +337,7 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
@Override
public void comment(char[] ch, int start, int length) throws SAXException {
- if (writeMode) {
+ if (!skipping) {
try {
flushText();
startChildInParent(cnb);
@@ -379,7 +396,7 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
public void startEntity(String name) throws SAXException {
}
- public void setChildPathSteps(List<SequenceType> childSeq) {
+ private void setChildPathSteps(List<SequenceType> childSeq) {
// this.childSeq = childSeq;
if (!childSeq.isEmpty()) {
subElement = new boolean[childSeq.size()];
@@ -398,10 +415,7 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
}
}
- public void setupElementWriter(ByteBuffer frame, FrameTupleAppender appender, IFrameWriter writer,
- FrameTupleAccessor fta, int t) throws IOException {
- this.frame = frame;
- this.appender = appender;
+ public void setupElementWriter(IFrameWriter writer, FrameTupleAccessor fta, int t) {
this.writer = writer;
this.fta = fta;
this.t = t;
@@ -430,7 +444,7 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
}
tvp.set(resultABVS.getByteArray(), resultABVS.getStartOffset(), resultABVS.getLength());
addNodeToTuple(tvp, t);
- writeMode = false;
+ skipping = true;
}
public void writeDocument(ArrayBackedValueStorage abvs) throws IOException {
@@ -470,8 +484,8 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
startChildInParent(anb, false);
}
- private void startChildInParent(AbstractNodeBuilder anb, boolean track) throws IOException {
- if (track) {
+ private void startChildInParent(AbstractNodeBuilder anb, boolean startNewElement) throws IOException {
+ if (startNewElement) {
elementABVS.reset();
anb.reset(elementABVS);
} else if (enbStack.isEmpty()) {
@@ -521,8 +535,8 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
/**
* Determines if the correct path step is active.
*/
- private boolean foundChildPathStep() {
- if (subElement.length != depth) {
+ private boolean foundFirstNonSkippedElement() {
+ if (subElement == null || subElement.length != depth) {
// Not the correct depth.
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/8b86884a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
index edef1a1..c1fd6f0 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
@@ -20,9 +20,11 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.List;
import java.util.zip.GZIPInputStream;
+import org.apache.vxquery.context.StaticContext;
import org.apache.vxquery.exceptions.VXQueryFileNotFoundException;
import org.apache.vxquery.exceptions.VXQueryParseException;
import org.apache.vxquery.types.SequenceType;
@@ -43,9 +45,23 @@ public class XMLParser {
final InputSource in;
public XMLParser(boolean attachTypes, ITreeNodeIdProvider idProvider) throws HyracksDataException {
+ this(attachTypes, idProvider, null, null, null, null);
+ }
+
+ public XMLParser(boolean attachTypes, ITreeNodeIdProvider idProvider, ByteBuffer frame,
+ FrameTupleAppender appender, List<Integer> childSeq, StaticContext staticContext)
+ throws HyracksDataException {
try {
parser = XMLReaderFactory.createXMLReader();
- handler = new SAXContentHandler(attachTypes, idProvider);
+ if (frame == null || appender == null) {
+ handler = new SAXContentHandler(attachTypes, idProvider);
+ } else {
+ List<SequenceType> childSequenceTypes = new ArrayList<SequenceType>();
+ for (int typeCode : childSeq) {
+ childSequenceTypes.add(staticContext.lookupSequenceType(typeCode));
+ }
+ handler = new SAXContentHandler(attachTypes, idProvider, frame, appender, childSequenceTypes);
+ }
parser.setContentHandler(handler);
parser.setProperty("http://xml.org/sax/properties/lexical-handler", handler);
in = new InputSource();
@@ -72,16 +88,15 @@ public class XMLParser {
}
}
- public void parseOutElements(File file, ByteBuffer frame, FrameTupleAppender appender, IFrameWriter writer,
- FrameTupleAccessor fta, int t, List<SequenceType> childSeq) throws HyracksDataException {
+ public void parseOutElements(File file, IFrameWriter writer, FrameTupleAccessor fta, int t)
+ throws HyracksDataException {
try {
if (file.getName().toLowerCase().endsWith(".xml.gz")) {
in.setCharacterStream(new InputStreamReader(new GZIPInputStream(new FileInputStream(file))));
} else {
in.setCharacterStream(new InputStreamReader(new FileInputStream(file)));
}
- handler.setChildPathSteps(childSeq);
- handler.setupElementWriter(frame, appender, writer, fta, t);
+ handler.setupElementWriter(writer, fta, t);
parser.parse(in);
} catch (FileNotFoundException e) {
throw new VXQueryFileNotFoundException(e, file);