You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by xi...@apache.org on 2017/09/11 21:20:49 UTC
asterixdb git commit: [ASTERIXDB-2086][RT] Remove table size from
InMemoryHashJoin interface
Repository: asterixdb
Updated Branches:
refs/heads/master a281eeaca -> c281d15de
[ASTERIXDB-2086][RT] Remove table size from InMemoryHashJoin interface
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
As this info is available in HashTable already, table size should not
be part of the interface.
Change-Id: I02a677ecfef80ccd4332447a6dcb5d480be6fe80
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1994
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <bu...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/c281d15d
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/c281d15d
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/c281d15d
Branch: refs/heads/master
Commit: c281d15deae4edae5f1b8fb4e1ad79e796deb399
Parents: a281eea
Author: Xikui Wang <xk...@gmail.com>
Authored: Sun Sep 10 12:29:56 2017 -0700
Committer: Xikui Wang <xk...@gmail.com>
Committed: Mon Sep 11 14:20:12 2017 -0700
----------------------------------------------------------------------
.../join/HybridHashJoinOperatorDescriptor.java | 17 ++++++++-------
.../dataflow/std/join/InMemoryHashJoin.java | 16 ++++++--------
.../InMemoryHashJoinOperatorDescriptor.java | 23 ++++----------------
.../std/join/OptimizedHybridHashJoin.java | 2 +-
...timizedHybridHashJoinOperatorDescriptor.java | 2 +-
.../std/structures/ISerializableTable.java | 6 +++++
.../structures/SimpleSerializableHashTable.java | 4 ++++
7 files changed, 32 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c281d15d/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java
index d9e6180..665bb2b 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java
@@ -309,10 +309,10 @@ public class HybridHashJoinOperatorDescriptor extends AbstractOperatorDescriptor
.createPartitioner();
int tableSize = (int) (state.memoryForHashtable * recordsPerFrame * factor);
ISerializableTable table = new SimpleSerializableHashTable(tableSize, ctx);
- state.joiner = new InMemoryHashJoin(ctx, tableSize, new FrameTupleAccessor(rd0), hpc0,
- new FrameTupleAccessor(rd1), rd1, hpc1,
- new FrameTuplePairComparator(keys0, keys1, comparators), isLeftOuter, nullWriters1, table,
- predEvaluator, null);
+ state.joiner =
+ new InMemoryHashJoin(ctx, new FrameTupleAccessor(rd0), hpc0, new FrameTupleAccessor(rd1),
+ rd1, hpc1, new FrameTuplePairComparator(keys0, keys1, comparators), isLeftOuter,
+ nullWriters1, table, predEvaluator, null);
bufferForPartitions = new IFrame[state.nPartitions];
state.fWriters = new RunFileWriter[state.nPartitions];
for (int i = 0; i < state.nPartitions; i++) {
@@ -505,10 +505,11 @@ public class HybridHashJoinOperatorDescriptor extends AbstractOperatorDescriptor
continue;
}
table.reset();
- InMemoryHashJoin joiner = new InMemoryHashJoin(ctx, tableSize,
- new FrameTupleAccessor(rd0), hpcRep0, new FrameTupleAccessor(rd1), rd1, hpcRep1,
- new FrameTuplePairComparator(keys0, keys1, comparators), isLeftOuter,
- nullWriters1, table, predEvaluator, null);
+ InMemoryHashJoin joiner =
+ new InMemoryHashJoin(ctx, new FrameTupleAccessor(rd0), hpcRep0,
+ new FrameTupleAccessor(rd1), rd1, hpcRep1,
+ new FrameTuplePairComparator(keys0, keys1, comparators), isLeftOuter,
+ nullWriters1, table, predEvaluator, null);
if (buildWriter != null) {
RunFileReader buildReader = buildWriter.createDeleteOnCloseReader();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c281d15d/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/InMemoryHashJoin.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/InMemoryHashJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/InMemoryHashJoin.java
index ec1c3a9..7f34d13 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/InMemoryHashJoin.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/InMemoryHashJoin.java
@@ -56,7 +56,6 @@ public class InMemoryHashJoin {
private final boolean isLeftOuter;
private final ArrayTupleBuilder missingTupleBuild;
private final ISerializableTable table;
- private final int tableSize;
private final TuplePointer storedTuplePointer;
private final boolean reverseOutputOrder; //Should we reverse the order of tuples, we are writing in output
private final IPredicateEvaluator predEvaluator;
@@ -67,23 +66,22 @@ public class InMemoryHashJoin {
private static final Logger LOGGER = Logger.getLogger(InMemoryHashJoin.class.getName());
- public InMemoryHashJoin(IHyracksTaskContext ctx, int tableSize, FrameTupleAccessor accessorProbe,
+ public InMemoryHashJoin(IHyracksTaskContext ctx, FrameTupleAccessor accessorProbe,
ITuplePartitionComputer tpcProbe, FrameTupleAccessor accessorBuild, RecordDescriptor rDBuild,
ITuplePartitionComputer tpcBuild, FrameTuplePairComparator comparator, boolean isLeftOuter,
IMissingWriter[] missingWritersBuild, ISerializableTable table, IPredicateEvaluator predEval,
ISimpleFrameBufferManager bufferManager)
throws HyracksDataException {
- this(ctx, tableSize, accessorProbe, tpcProbe, accessorBuild, rDBuild, tpcBuild, comparator, isLeftOuter,
+ this(ctx, accessorProbe, tpcProbe, accessorBuild, rDBuild, tpcBuild, comparator, isLeftOuter,
missingWritersBuild, table, predEval, false, bufferManager);
}
- public InMemoryHashJoin(IHyracksTaskContext ctx, int tableSize, FrameTupleAccessor accessorProbe,
+ public InMemoryHashJoin(IHyracksTaskContext ctx, FrameTupleAccessor accessorProbe,
ITuplePartitionComputer tpcProbe, FrameTupleAccessor accessorBuild,
RecordDescriptor rDBuild, ITuplePartitionComputer tpcBuild, FrameTuplePairComparator comparator,
boolean isLeftOuter, IMissingWriter[] missingWritersBuild, ISerializableTable table,
IPredicateEvaluator predEval, boolean reverse, ISimpleFrameBufferManager bufferManager)
throws HyracksDataException {
- this.tableSize = tableSize;
this.table = table;
storedTuplePointer = new TuplePointer();
buffers = new ArrayList<>();
@@ -109,12 +107,12 @@ public class InMemoryHashJoin {
reverseOutputOrder = reverse;
this.tupleAccessor = new TupleInFrameListAccessor(rDBuild, buffers);
this.bufferManager = bufferManager;
- if (tableSize != 0) {
+ if (table.getTableSize() != 0) {
isTableCapacityNotZero = true;
} else {
isTableCapacityNotZero = false;
}
- LOGGER.fine("InMemoryHashJoin has been created for a table size of " + tableSize + " for Thread ID "
+ LOGGER.fine("InMemoryHashJoin has been created for a table size of " + table.getTableSize() + " for Thread ID "
+ Thread.currentThread().getId() + ".");
}
@@ -124,7 +122,7 @@ public class InMemoryHashJoin {
accessorBuild.reset(buffer);
int tCount = accessorBuild.getTupleCount();
for (int i = 0; i < tCount; ++i) {
- int entry = tpcBuild.partition(accessorBuild, i, tableSize);
+ int entry = tpcBuild.partition(accessorBuild, i, table.getTableSize());
storedTuplePointer.reset(bIndex, i);
// If an insertion fails, then tries to insert the same tuple pointer again after compacting the table.
if (!table.insert(entry, storedTuplePointer)) {
@@ -160,7 +158,7 @@ public class InMemoryHashJoin {
void join(int tid, IFrameWriter writer) throws HyracksDataException {
boolean matchFound = false;
if (isTableCapacityNotZero) {
- int entry = tpcProbe.partition(accessorProbe, tid, tableSize);
+ int entry = tpcProbe.partition(accessorProbe, tid, table.getTableSize());
int tupleCount = table.getTupleCount(entry);
for (int i = 0; i < tupleCount; i++) {
table.getTuplePointer(entry, i, storedTuplePointer);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c281d15d/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java
index 702dae6..d81d955 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java
@@ -104,21 +104,6 @@ public class InMemoryHashJoinOperatorDescriptor extends AbstractOperatorDescript
this.memSizeInFrames = memSizeInFrames;
}
- public InMemoryHashJoinOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] keys0, int[] keys1,
- IBinaryHashFunctionFactory[] hashFunctionFactories, IBinaryComparatorFactory[] comparatorFactories,
- RecordDescriptor recordDescriptor, int tableSize, int memSizeInFrames) {
- this(spec, keys0, keys1, hashFunctionFactories, comparatorFactories, recordDescriptor, tableSize, null,
- memSizeInFrames);
- }
-
- public InMemoryHashJoinOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] keys0, int[] keys1,
- IBinaryHashFunctionFactory[] hashFunctionFactories, IBinaryComparatorFactory[] comparatorFactories,
- RecordDescriptor recordDescriptor, boolean isLeftOuter, IMissingWriterFactory[] nullWriterFactories1,
- int tableSize, int memSizeInFrames) {
- this(spec, keys0, keys1, hashFunctionFactories, comparatorFactories, null, recordDescriptor, isLeftOuter,
- nullWriterFactories1, tableSize, memSizeInFrames);
- }
-
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
ActivityId hbaId = new ActivityId(odId, 0);
@@ -203,10 +188,10 @@ public class InMemoryHashJoinOperatorDescriptor extends AbstractOperatorDescript
state = new HashBuildTaskState(ctx.getJobletContext().getJobId(),
new TaskId(getActivityId(), partition));
ISerializableTable table = new SerializableHashTable(tableSize, ctx, bufferManager);
- state.joiner = new InMemoryHashJoin(ctx, tableSize, new FrameTupleAccessor(rd0), hpc0,
- new FrameTupleAccessor(rd1), rd1, hpc1,
- new FrameTuplePairComparator(keys0, keys1, comparators), isLeftOuter, nullWriters1, table,
- predEvaluator, bufferManager);
+ state.joiner =
+ new InMemoryHashJoin(ctx, new FrameTupleAccessor(rd0), hpc0, new FrameTupleAccessor(rd1),
+ rd1, hpc1, new FrameTuplePairComparator(keys0, keys1, comparators), isLeftOuter,
+ nullWriters1, table, predEvaluator, bufferManager);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c281d15d/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
index 5de272a..d49a6dd 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
@@ -452,7 +452,7 @@ public class OptimizedHybridHashJoin {
private void createInMemoryJoiner(int inMemTupCount) throws HyracksDataException {
ISerializableTable table = new SerializableHashTable(inMemTupCount, ctx, bufferManagerForHashTable);
- this.inMemJoiner = new InMemoryHashJoin(ctx, inMemTupCount, new FrameTupleAccessor(probeRd), probeHpc,
+ this.inMemJoiner = new InMemoryHashJoin(ctx, new FrameTupleAccessor(probeRd), probeHpc,
new FrameTupleAccessor(buildRd), buildRd, buildHpc,
new FrameTuplePairComparator(probeKeys, buildKeys, comparators), isLeftOuter, nonMatchWriters, table,
predEvaluator, isReversed, bufferManagerForHashTable);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c281d15d/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
index c44c583..c795144 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
@@ -717,7 +717,7 @@ public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorD
ISimpleFrameBufferManager bufferManager = new FramePoolBackedFrameBufferManager(framePool);
ISerializableTable table = new SerializableHashTable(tabSize, ctx, bufferManager);
- InMemoryHashJoin joiner = new InMemoryHashJoin(ctx, tabSize, new FrameTupleAccessor(probeRDesc),
+ InMemoryHashJoin joiner = new InMemoryHashJoin(ctx, new FrameTupleAccessor(probeRDesc),
hpcRepProbe, new FrameTupleAccessor(buildRDesc), buildRDesc, hpcRepBuild,
new FrameTuplePairComparator(pKeys, bKeys, comparators), isLeftOuter, nonMatchWriter, table,
predEvaluator, isReversed, bufferManager);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c281d15d/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/structures/ISerializableTable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/structures/ISerializableTable.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/structures/ISerializableTable.java
index d0e0616..015ddb3 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/structures/ISerializableTable.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/structures/ISerializableTable.java
@@ -64,4 +64,10 @@ public interface ISerializableTable {
* Prints out the internal information of this table.
*/
String printInfo();
+
+ /**
+ * Returns the number of entries of this table.
+ * @return the number entries.
+ */
+ int getTableSize();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c281d15d/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/structures/SimpleSerializableHashTable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/structures/SimpleSerializableHashTable.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/structures/SimpleSerializableHashTable.java
index b1d1f27..6497a53 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/structures/SimpleSerializableHashTable.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/structures/SimpleSerializableHashTable.java
@@ -539,4 +539,8 @@ public class SimpleSerializableHashTable implements ISerializableTable {
return null;
}
+ @Override
+ public int getTableSize() {
+ return tableSize;
+ }
}