You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2014/04/02 23:13:20 UTC
[1/2] git commit: Updated the cluster details to consider the number
of local processors.
Repository: incubator-vxquery
Updated Branches:
refs/heads/prestonc/hash_join 0b80d3f56 -> c813b097f
Updated the cluster details to consider the number of local processors.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/903576e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/903576e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/903576e1
Branch: refs/heads/prestonc/hash_join
Commit: 903576e11361a91d2b5ea7e2f9f8cf7af7b8680f
Parents: 0b80d3f
Author: Preston Carman <pr...@apache.org>
Authored: Tue Apr 1 21:52:00 2014 -0700
Committer: Preston Carman <pr...@apache.org>
Committed: Tue Apr 1 21:52:00 2014 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/vxquery/cli/VXQuery.java | 1 -
.../vxquery/metadata/VXQueryMetadataProvider.java | 13 +++++++++++--
.../vxquery/xmlquery/query/XMLQueryCompiler.java | 9 +++++----
3 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/903576e1/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
----------------------------------------------------------------------
diff --git a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
index 3f0975a..ada055d 100644
--- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
+++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
@@ -33,7 +33,6 @@ import org.apache.vxquery.compiler.algebricks.prettyprint.VXQueryLogicalExpressi
import org.apache.vxquery.context.DynamicContext;
import org.apache.vxquery.context.DynamicContextImpl;
import org.apache.vxquery.context.RootStaticContextImpl;
-import org.apache.vxquery.context.StaticContext;
import org.apache.vxquery.context.StaticContextImpl;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.result.ResultUtils;
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/903576e1/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
index 71963be..0aaac5c 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
@@ -52,7 +52,7 @@ import edu.uci.ics.hyracks.dataflow.std.result.ResultWriterOperatorDescriptor;
public class VXQueryMetadataProvider implements IMetadataProvider<String, String> {
String[] nodeList;
- public void setNodeList(String[] nodeList) {
+ public VXQueryMetadataProvider(String[] nodeList) {
this.nodeList = nodeList;
}
@@ -74,7 +74,15 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(scanner, constraint);
}
- public AlgebricksPartitionConstraint getClusterLocations(String[] nodeList, int partitions) {
+ public AlgebricksPartitionConstraint getClusterLocations() {
+ int availableProcessors = Runtime.getRuntime().availableProcessors();
+ if (availableProcessors < 1) {
+ availableProcessors = 1;
+ }
+ return getClusterLocations(nodeList, availableProcessors);
+ }
+
+ private static AlgebricksPartitionConstraint getClusterLocations(String[] nodeList, int partitions) {
ArrayList<String> locs = new ArrayList<String>();
for (String node : nodeList) {
for (int j = 0; j < partitions; j++) {
@@ -152,6 +160,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
public FunctionIdentifier getFunctionIdentifier() {
return fid;
}
+
public boolean isFunctional() {
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/903576e1/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
index 54bac7e..d07de9a 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
@@ -39,8 +39,10 @@ import org.apache.vxquery.types.Quantifier;
import org.apache.vxquery.types.SequenceType;
import org.apache.vxquery.xmlquery.ast.ModuleNode;
import org.apache.vxquery.xmlquery.translator.XMLQueryTranslator;
+import org.omg.SendingContext.RunTime;
import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.compiler.api.HeuristicCompilerFactoryBuilder;
@@ -78,7 +80,7 @@ public class XMLQueryCompiler {
private final ICompilerFactory cFactory;
private final VXQueryMetadataProvider mdProvider;
-
+
private LogicalOperatorPrettyPrintVisitor pprinter;
private ModuleNode moduleNode;
@@ -145,10 +147,9 @@ public class XMLQueryCompiler {
}
});
builder.setNullWriterFactory(new VXQueryNullWriterFactory());
- builder.setClusterLocations(new AlgebricksAbsolutePartitionConstraint(nodeList));
+ mdProvider = new VXQueryMetadataProvider(nodeList);
+ builder.setClusterLocations(mdProvider.getClusterLocations());
cFactory = builder.create();
- mdProvider = new VXQueryMetadataProvider();
- mdProvider.setNodeList(nodeList);
}
public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel)
[2/2] git commit: Simplified the aggregate step functions with better
init values.
Posted by pr...@apache.org.
Simplified the aggregate step functions with better init values.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/c813b097
Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/c813b097
Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/c813b097
Branch: refs/heads/prestonc/hash_join
Commit: c813b097f8179771a934a9aac0cce0824edf5986
Parents: 903576e
Author: Preston Carman <pr...@apache.org>
Authored: Tue Apr 1 21:57:19 2014 -0700
Committer: Preston Carman <pr...@apache.org>
Committed: Tue Apr 1 21:57:19 2014 -0700
----------------------------------------------------------------------
.../AvgLocalAggregateEvaluatorFactory.java | 23 ++++------
.../FnSumAggregateEvaluatorFactory.java | 45 ++++++--------------
2 files changed, 21 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/c813b097/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/aggregate/AvgLocalAggregateEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/aggregate/AvgLocalAggregateEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/aggregate/AvgLocalAggregateEvaluatorFactory.java
index 45414dc..6763a67 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/aggregate/AvgLocalAggregateEvaluatorFactory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/aggregate/AvgLocalAggregateEvaluatorFactory.java
@@ -17,13 +17,11 @@
package org.apache.vxquery.runtime.functions.aggregate;
import java.io.DataOutput;
-import java.io.IOException;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
import org.apache.vxquery.datamodel.values.ValueTag;
import org.apache.vxquery.datamodel.values.XDMConstants;
-import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.runtime.functions.arithmetic.AddOperation;
import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentAggregateEvaluator;
@@ -62,6 +60,14 @@ public class AvgLocalAggregateEvaluatorFactory extends AbstractTaggedValueArgume
@Override
public void init() throws AlgebricksException {
count = 0;
+ try {
+ abvsSum.reset();
+ dOutSum.write(ValueTag.XS_INTEGER_TAG);
+ dOutSum.writeLong(0);
+ tvpSum.set(abvsSum);
+ } catch (Exception e) {
+ throw new AlgebricksException(e);
+ }
}
@Override
@@ -92,18 +98,7 @@ public class AvgLocalAggregateEvaluatorFactory extends AbstractTaggedValueArgume
@Override
protected void step(TaggedValuePointable[] args) throws SystemException {
TaggedValuePointable tvp = args[0];
- if (count == 0) {
- // Init.
- try {
- abvsSum.reset();
- dOutSum.write(tvp.getByteArray(), tvp.getStartOffset(), tvp.getLength());
- tvpSum.set(abvsSum);
- } catch (IOException e) {
- throw new SystemException(ErrorCode.SYSE0001, e.toString());
- }
- } else {
- FunctionHelper.arithmeticOperation(aOp, dCtx, tvp, tvpSum, tvpSum);
- }
+ FunctionHelper.arithmeticOperation(aOp, dCtx, tvp, tvpSum, tvpSum);
count++;
}
};
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/c813b097/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/aggregate/FnSumAggregateEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/aggregate/FnSumAggregateEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/aggregate/FnSumAggregateEvaluatorFactory.java
index 8572a11..8a55f3c 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/aggregate/FnSumAggregateEvaluatorFactory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/aggregate/FnSumAggregateEvaluatorFactory.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
import org.apache.vxquery.datamodel.values.ValueTag;
-import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.runtime.functions.arithmetic.AddOperation;
import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentAggregateEvaluator;
@@ -44,56 +43,36 @@ public class FnSumAggregateEvaluatorFactory extends AbstractTaggedValueArgumentA
@Override
protected IAggregateEvaluator createEvaluator(IScalarEvaluator[] args) throws AlgebricksException {
- final ArrayBackedValueStorage abvsCount = new ArrayBackedValueStorage();
- final DataOutput dOutCount = abvsCount.getDataOutput();
final ArrayBackedValueStorage abvsSum = new ArrayBackedValueStorage();
final DataOutput dOutSum = abvsSum.getDataOutput();
final AddOperation aOp = new AddOperation();
return new AbstractTaggedValueArgumentAggregateEvaluator(args) {
- long count;
TaggedValuePointable tvpSum = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
+ // TODO Check if the second argument is supplied as the zero value.
+
@Override
public void init() throws AlgebricksException {
- count = 0;
+ try {
+ abvsSum.reset();
+ dOutSum.write(ValueTag.XS_INTEGER_TAG);
+ dOutSum.writeLong(0);
+ tvpSum.set(abvsSum);
+ } catch (IOException e) {
+ throw new AlgebricksException(e.toString());
+ }
}
@Override
public void finish(IPointable result) throws AlgebricksException {
- // TODO What is returned when step is never called. Since the second argument is the zero value.
- if (count == 0) {
- // No argument return an integer.
- try {
- abvsCount.reset();
- dOutCount.write(ValueTag.XS_INTEGER_TAG);
- dOutCount.writeLong(0);
- result.set(abvsCount);
- } catch (Exception e) {
-
- throw new AlgebricksException(e);
- }
- } else {
- result.set(tvpSum);
- }
+ result.set(tvpSum);
}
@Override
protected void step(TaggedValuePointable[] args) throws SystemException {
TaggedValuePointable tvp = args[0];
- if (count == 0) {
- // Init.
- try {
- abvsSum.reset();
- dOutSum.write(tvp.getByteArray(), tvp.getStartOffset(), tvp.getLength());
- tvpSum.set(abvsSum);
- } catch (IOException e) {
- throw new SystemException(ErrorCode.SYSE0001, e.toString());
- }
- } else {
- FunctionHelper.arithmeticOperation(aOp, dCtx, tvp, tvpSum, tvpSum);
- }
- count++;
+ FunctionHelper.arithmeticOperation(aOp, dCtx, tvp, tvpSum, tvpSum);
}
};
}