You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/06/25 21:07:50 UTC
[01/14] git commit: DRILL-938: Set the links entry correctly when
resizing when hash chains cross batch boundaries.
Repository: incubator-drill
Updated Branches:
refs/heads/master 79c1502c1 -> 33c28f624
DRILL-938: Set the links entry correctly when resizing when hash chains cross batch boundaries.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0c9d4ebb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0c9d4ebb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0c9d4ebb
Branch: refs/heads/master
Commit: 0c9d4ebbf5e0c2a5f9b3a554f4a051568efa46e7
Parents: 79c1502
Author: Aman Sinha <as...@maprtech.com>
Authored: Fri Jun 20 00:06:07 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:08:14 2014 -0700
----------------------------------------------------------------------
.../physical/impl/common/HashTableTemplate.java | 84 ++++++++++++--------
1 file changed, 53 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0c9d4ebb/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
index f7cadf1..099d2ee 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
@@ -33,6 +33,7 @@ import org.apache.drill.exec.compile.sig.RuntimeOverridden;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.holders.IntHolder;
import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.exec.vector.IntVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
@@ -41,8 +42,9 @@ public abstract class HashTableTemplate implements HashTable {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashTable.class);
private static final boolean EXTRA_DEBUG = false;
-
+
private static final int EMPTY_SLOT = -1;
+ // private final int MISSING_VALUE = 65544;
// A hash 'bucket' consists of the start index to indicate start of a hash chain
@@ -108,8 +110,12 @@ public abstract class HashTableTemplate implements HashTable {
private int maxOccupiedIdx = -1;
private int batchOutputCount = 0;
+ private int batchIndex = 0;
+
private BatchHolder(int idx) {
+ this.batchIndex = idx;
+
if (idx == 0) { // first batch holder can use the original htContainer
htContainer = htContainerOrig;
} else { // otherwise create a new one using the original's fields
@@ -199,19 +205,19 @@ public abstract class HashTableTemplate implements HashTable {
private void rehash(int numbuckets, IntVector newStartIndices, int batchStartIdx) {
- if (EXTRA_DEBUG) logger.debug("Resizing and rehashing table...");
+ logger.debug("Rehashing entries within the batch: {}; batchStartIdx = {}, total numBuckets in hash table = {}.", batchIndex, batchStartIdx, numbuckets);
int size = links.getAccessor().getValueCount();
IntVector newLinks = allocMetadataVector(size, EMPTY_SLOT);
IntVector newHashValues = allocMetadataVector(size, 0);
-
+
for (int i = 0; i <= maxOccupiedIdx; i++) {
int entryIdxWithinBatch = i;
int entryIdx = entryIdxWithinBatch + batchStartIdx;
int hash = hashValues.getAccessor().get(entryIdxWithinBatch); // get the already saved hash value
int bucketIdx = getBucketIndex(hash, numbuckets);
int newStartIdx = newStartIndices.getAccessor().get(bucketIdx);
-
+
if (newStartIdx == EMPTY_SLOT) { // new bucket was empty
newStartIndices.getMutator().setSafe(bucketIdx, entryIdx); // update the start index to point to entry
newLinks.getMutator().setSafe(entryIdxWithinBatch, EMPTY_SLOT);
@@ -220,12 +226,19 @@ public abstract class HashTableTemplate implements HashTable {
if (EXTRA_DEBUG) logger.debug("New bucket was empty. bucketIdx = {}, newStartIndices[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, bucketIdx, newStartIndices.getAccessor().get(bucketIdx), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
} else {
- // follow the new table's hash chain until we encounter empty slot
+ // follow the new table's hash chain until we encounter empty slot. Note that the hash chain could
+ // traverse multiple batch holders, so make sure we are accessing the right batch holder.
int idx = newStartIdx;
int idxWithinBatch = 0;
+ BatchHolder bh = this;
while (true) {
- idxWithinBatch = idx & BATCH_MASK;
- if (newLinks.getAccessor().get(idxWithinBatch) == EMPTY_SLOT) {
+ if (idx != EMPTY_SLOT) {
+ idxWithinBatch = idx & BATCH_MASK;
+ int batchIdx = ((idx >>> 16) & BATCH_MASK);
+ bh = batchHolders.get(batchIdx);
+ }
+
+ if (bh == this && newLinks.getAccessor().get(idxWithinBatch) == EMPTY_SLOT) {
newLinks.getMutator().setSafe(idxWithinBatch, entryIdx);
newLinks.getMutator().setSafe(entryIdxWithinBatch, EMPTY_SLOT);
newHashValues.getMutator().setSafe(entryIdxWithinBatch, hash);
@@ -233,8 +246,20 @@ public abstract class HashTableTemplate implements HashTable {
if (EXTRA_DEBUG) logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch, newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
break;
+ } else if (bh != this && bh.links.getAccessor().get(idxWithinBatch) == EMPTY_SLOT) {
+ bh.links.getMutator().setSafe(idxWithinBatch, entryIdx); // update the link in the other batch
+ newLinks.getMutator().setSafe(entryIdxWithinBatch, EMPTY_SLOT); // update the newLink entry in this batch to mark end of the hash chain
+ newHashValues.getMutator().setSafe(entryIdxWithinBatch, hash);
+
+ if (EXTRA_DEBUG) logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch, newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
+
+ break;
+ }
+ if (bh == this) {
+ idx = newLinks.getAccessor().get(idxWithinBatch);
+ } else {
+ idx = bh.links.getAccessor().get(idxWithinBatch);
}
- idx = newLinks.getAccessor().get(idxWithinBatch);
}
}
@@ -251,14 +276,8 @@ public abstract class HashTableTemplate implements HashTable {
private boolean outputKeys() {
/** for debugging
- Object tmp = (htContainer).getValueAccessorById(0, BigIntVector.class).getValueVector();
- BigIntVector vv0 = null;
- BigIntHolder holder = null;
-
- if (tmp != null) {
- vv0 = ((BigIntVector) tmp);
- holder = new BigIntHolder();
- }
+ BigIntVector vv0 = getValueVector(0);
+ BigIntHolder holder = new BigIntHolder();
*/
for (int i = 0; i <= maxOccupiedIdx; i++) {
@@ -296,6 +315,17 @@ public abstract class HashTableTemplate implements HashTable {
links.clear();
hashValues.clear();
}
+
+ // Only used for internal debugging. Get the value vector at a particular index from the htContainer.
+ // By default this assumes the VV is a BigIntVector.
+ private ValueVector getValueVector(int index) {
+ Object tmp = (htContainer).getValueAccessorById(BigIntVector.class, index).getValueVector();
+ if (tmp != null) {
+ BigIntVector vv0 = ((BigIntVector) tmp);
+ return vv0;
+ }
+ return null;
+ }
// These methods will be code-generated
@@ -414,6 +444,7 @@ public abstract class HashTableTemplate implements HashTable {
public PutStatus put(int incomingRowIdx, IntHolder htIdxHolder) {
int hash = getHashBuild(incomingRowIdx);
+ hash = Math.abs(hash);
int i = getBucketIndex(hash, numBuckets());
int startIdx = startIndices.getAccessor().get(i);
int currentIdx;
@@ -510,12 +541,14 @@ public abstract class HashTableTemplate implements HashTable {
@Override
public int containsKey(int incomingRowIdx, boolean isProbe) {
int hash = isProbe ? getHashProbe(incomingRowIdx) : getHashBuild(incomingRowIdx);
+ hash = Math.abs(hash);
int i = getBucketIndex(hash, numBuckets());
int currentIdx = startIndices.getAccessor().get(i);
-
- if (currentIdx == EMPTY_SLOT)
- return -1;
+
+ if (currentIdx == EMPTY_SLOT) {
+ return -1;
+ }
BatchHolder bh = batchHolders.get( (currentIdx >>> 16) & BATCH_MASK);
currentIdxHolder.value = currentIdx;
@@ -529,7 +562,7 @@ public abstract class HashTableTemplate implements HashTable {
} else if (currentIdxHolder.value == EMPTY_SLOT) {
break;
} else {
- bh = batchHolders.get( (currentIdxHolder.value >>> 16) & BATCH_MASK);
+ bh = batchHolders.get( (currentIdxHolder.value >>> 16) & BATCH_MASK);
}
}
@@ -611,17 +644,6 @@ public abstract class HashTableTemplate implements HashTable {
numResizing++;
}
- /*
- public boolean outputKeys() {
- for (BatchHolder bh : batchHolders) {
- if ( ! bh.outputKeys()) {
- return false;
- }
- }
- return true;
- }
- */
-
public boolean outputKeys(int batchIdx) {
assert batchIdx < batchHolders.size();
if (! batchHolders.get(batchIdx).outputKeys()) {
[04/14] git commit: DRILL-1070: Update work manager to capture errors
in Foreman and Fragment threads rather than drop to JVM.
Posted by ja...@apache.org.
DRILL-1070: Update work manager to capture errors in Foreman and Fragment threads rather than drop to JVM.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/bfb422ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/bfb422ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/bfb422ed
Branch: refs/heads/master
Commit: bfb422ed63597b99e875d13e50764335a29a8a11
Parents: 16808f4
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Jun 21 11:25:19 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:09:17 2014 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/work/WorkManager.java | 138 +++++++++++--------
1 file changed, 84 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/bfb422ed/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
index bddc1cf..7d5d37f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
@@ -25,7 +25,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.drill.exec.cache.DistributedCache;
@@ -33,6 +33,7 @@ import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.rpc.NamedThreadFactory;
import org.apache.drill.exec.rpc.control.Controller;
import org.apache.drill.exec.rpc.control.WorkEventBus;
@@ -55,12 +56,13 @@ import com.codahale.metrics.MetricRegistry;
import com.google.common.collect.Maps;
import com.google.common.collect.Queues;
-public class WorkManager implements Closeable{
+public class WorkManager implements Closeable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WorkManager.class);
- private Set<FragmentManager> incomingFragments = Collections.newSetFromMap(Maps.<FragmentManager, Boolean> newConcurrentMap());
+ private Set<FragmentManager> incomingFragments = Collections.newSetFromMap(Maps
+ .<FragmentManager, Boolean> newConcurrentMap());
- private PriorityBlockingQueue<Runnable> pendingTasks = Queues.newPriorityBlockingQueue();
+ private LinkedBlockingQueue<RunnableWrapper> pendingTasks = Queues.newLinkedBlockingQueue();
private Map<FragmentHandle, FragmentExecutor> runningFragments = Maps.newConcurrentMap();
@@ -79,7 +81,7 @@ public class WorkManager implements Closeable{
private ExecutorService executor;
private final EventThread eventThread;
- public WorkManager(BootStrapContext context){
+ public WorkManager(BootStrapContext context) {
this.bee = new WorkerBee();
this.workBus = new WorkEventBus(bee);
this.bContext = context;
@@ -89,28 +91,31 @@ public class WorkManager implements Closeable{
this.dataHandler = new DataResponseHandlerImpl(bee);
}
- public void start(DrillbitEndpoint endpoint, DistributedCache cache, Controller controller, DataConnectionCreator data, ClusterCoordinator coord, PStoreProvider provider){
+ public void start(DrillbitEndpoint endpoint, DistributedCache cache, Controller controller,
+ DataConnectionCreator data, ClusterCoordinator coord, PStoreProvider provider) {
this.dContext = new DrillbitContext(endpoint, bContext, coord, controller, data, cache, workBus, provider);
- // executor = Executors.newFixedThreadPool(dContext.getConfig().getInt(ExecConstants.EXECUTOR_THREADS)
+ // executor = Executors.newFixedThreadPool(dContext.getConfig().getInt(ExecConstants.EXECUTOR_THREADS)
executor = Executors.newCachedThreadPool(new NamedThreadFactory("WorkManager-"));
eventThread.start();
- dContext.getMetrics().register(MetricRegistry.name("drill.exec.work.running_fragments." + dContext.getEndpoint().getUserPort()),
+ dContext.getMetrics().register(
+ MetricRegistry.name("drill.exec.work.running_fragments." + dContext.getEndpoint().getUserPort()),
new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- return runningFragments.size();
- }
+ @Override
+ public Integer getValue() {
+ return runningFragments.size();
+ }
});
- dContext.getMetrics().register(MetricRegistry.name("drill.exec.work.pendingTasks" + dContext.getEndpoint().getUserPort()),
+ dContext.getMetrics().register(
+ MetricRegistry.name("drill.exec.work.pendingTasks" + dContext.getEndpoint().getUserPort()),
new Gauge<Integer>() {
- @Override
- public Integer getValue() {
- return pendingTasks.size();
- }
+ @Override
+ public Integer getValue() {
+ return pendingTasks.size();
+ }
});
}
- public WorkEventBus getWorkBus(){
+ public WorkEventBus getWorkBus() {
return workBus;
}
@@ -118,11 +123,11 @@ public class WorkManager implements Closeable{
return dataHandler;
}
- public ControlMessageHandler getControlMessageHandler(){
+ public ControlMessageHandler getControlMessageHandler() {
return controlMessageWorker;
}
- public UserWorker getUserWorker(){
+ public UserWorker getUserWorker() {
return userWorker;
}
@@ -141,43 +146,52 @@ public class WorkManager implements Closeable{
}
}
-
public DrillbitContext getContext() {
return dContext;
}
+ private static String getId(FragmentHandle handle){
+ return "FragmentExecutor: " + QueryIdHelper.getQueryId(handle.getQueryId()) + ':' + handle.getMajorFragmentId() + ':' + handle.getMinorFragmentId();
+ }
+
// create this so items can see the data here whether or not they are in this package.
- public class WorkerBee{
+ public class WorkerBee {
+
+
- public void addFragmentRunner(FragmentExecutor runner){
+ public void addFragmentRunner(FragmentExecutor runner) {
logger.debug("Adding pending task {}", runner);
- pendingTasks.add(runner);
+ RunnableWrapper wrapper = new RunnableWrapper(runner, getId(runner.getContext().getHandle()));
+ pendingTasks.add(wrapper);
}
- public void addNewForeman(Foreman foreman){
- pendingTasks.add(foreman);
+ public void addNewForeman(Foreman foreman) {
+ String id = "Foreman: " + QueryIdHelper.getQueryId(foreman.getQueryId());
+ RunnableWrapper wrapper = new RunnableWrapper(foreman, id);
+ pendingTasks.add(wrapper);
queries.put(foreman.getQueryId(), foreman);
}
-
- public void addFragmentPendingRemote(FragmentManager handler){
+ public void addFragmentPendingRemote(FragmentManager handler) {
incomingFragments.add(handler);
}
- public void startFragmentPendingRemote(FragmentManager handler){
+ public void startFragmentPendingRemote(FragmentManager handler) {
incomingFragments.remove(handler);
- pendingTasks.add(handler.getRunnable());
+ FragmentExecutor runner = handler.getRunnable();
+ RunnableWrapper wrapper = new RunnableWrapper(runner, getId(runner.getContext().getHandle()));
+ pendingTasks.add(wrapper);
}
- public FragmentExecutor getFragmentRunner(FragmentHandle handle){
+ public FragmentExecutor getFragmentRunner(FragmentHandle handle) {
return runningFragments.get(handle);
}
- public Foreman getForemanForQueryId(QueryId queryId){
+ public Foreman getForemanForQueryId(QueryId queryId) {
return queries.get(queryId);
}
- public void retireForeman(Foreman foreman){
+ public void retireForeman(Foreman foreman) {
queries.remove(foreman.getQueryId(), foreman);
}
@@ -187,34 +201,50 @@ public class WorkManager implements Closeable{
}
+ private class EventThread extends Thread {
+ public EventThread() {
+ this.setDaemon(true);
+ this.setName("WorkManager Event Thread");
+ }
-
- private class EventThread extends Thread{
- public EventThread(){
- this.setDaemon(true);
- this.setName("WorkManager Event Thread");
- }
-
- @Override
- public void run() {
- try {
- while(true){
-// logger.debug("Polling for pending work tasks.");
- Runnable r = pendingTasks.take();
- if(r != null){
- logger.debug("Starting pending task {}", r);
- executor.execute(r);
+ @Override
+ public void run() {
+ try {
+ while (true) {
+ // logger.debug("Polling for pending work tasks.");
+ Runnable r = pendingTasks.take();
+ if (r != null) {
+ logger.debug("Starting pending task {}", r);
+ executor.execute(r);
+ }
+
+ }
+ } catch (InterruptedException e) {
+ logger.info("Work Manager stopping as it was interrupted.");
}
-
- }
- } catch (InterruptedException e) {
- logger.info("Work Manager stopping as it was interrupted.");
}
+
}
+ private class RunnableWrapper implements Runnable {
- }
+ private final Runnable inner;
+ private final String id;
+ public RunnableWrapper(Runnable r, String id){
+ this.inner = r;
+ this.id = id;
+ }
+ @Override
+ public void run() {
+ try{
+ inner.run();
+ }catch(Exception | Error e){
+ logger.error("Failure while running wrapper [{}]", id, e);
+ }
+ }
+
+ }
}
[02/14] git commit: Small stats fix: add current Exchange to the
parent fragment stat.
Posted by ja...@apache.org.
Small stats fix: add current Exchange to the parent fragment stat.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/8773f84b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/8773f84b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/8773f84b
Branch: refs/heads/master
Commit: 8773f84b7b8b238f9653b05607e153438a624777
Parents: 0c9d4eb
Author: Aman Sinha <as...@maprtech.com>
Authored: Fri Jun 20 17:59:02 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:08:22 2014 -0700
----------------------------------------------------------------------
.../exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8773f84b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
index 168fd28..06b72f9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
@@ -45,6 +45,7 @@ public class ExcessiveExchangeIdentifier extends BasePrelVisitor<Prel, Excessive
@Override
public Prel visitExchange(ExchangePrel prel, MajorFragmentStat parent) throws RuntimeException {
+ parent.add(prel);
MajorFragmentStat newFrag = new MajorFragmentStat();
Prel newChild = ((Prel) prel.getChild()).accept(this, newFrag);
[05/14] git commit: DRILL-1071: Update local pstore provider to hold
data when running in write disabled mode. Allows use of web guy for profile
viewing while debugging.
Posted by ja...@apache.org.
DRILL-1071: Update local pstore provider to hold data when running in write disabled mode. Allows use of web guy for profile viewing while debugging.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0dfeac8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0dfeac8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0dfeac8f
Branch: refs/heads/master
Commit: 0dfeac8f8e8ad9cde3741b3a246d88bd11f0593e
Parents: bfb422e
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Jun 21 11:26:25 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:09:30 2014 -0700
----------------------------------------------------------------------
.../drill/exec/store/sys/local/LocalPStoreProvider.java | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dfeac8f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
index 16fd875..5d97b1b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.store.sys.local;
import java.io.File;
import java.io.IOException;
+import java.util.concurrent.ConcurrentMap;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
@@ -27,6 +28,8 @@ import org.apache.drill.exec.store.sys.PStoreConfig;
import org.apache.drill.exec.store.sys.PStoreProvider;
import org.apache.drill.exec.store.sys.PStoreRegistry;
+import com.google.common.collect.Maps;
+
/**
* A really simple provider that stores data in the local file system, one value per file.
*/
@@ -35,10 +38,14 @@ public class LocalPStoreProvider implements PStoreProvider{
private File path;
private final boolean enableWrite;
+ private ConcurrentMap<PStoreConfig<?>, PStore<?>> pstores;
public LocalPStoreProvider(DrillConfig config) {
path = new File(config.getString(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH));
enableWrite = config.getBoolean(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE);
+ if(!enableWrite){
+ pstores = Maps.newConcurrentMap();
+ }
}
public LocalPStoreProvider(PStoreRegistry registry) {
@@ -54,7 +61,10 @@ public class LocalPStoreProvider implements PStoreProvider{
if(enableWrite){
return new LocalPStore<V>(path, storeConfig);
}else{
- return new NoWriteLocalPStore<V>();
+ PStore<V> p = new NoWriteLocalPStore<V>();
+ PStore<?> p2 = pstores.putIfAbsent(storeConfig, p);
+ if(p2 != null) return (PStore<V>) p2;
+ return p;
}
}
[08/14] DRILL-1067: RPC layer improvements - Unfork Netty and upgrade
to 4.0.20.Final - Create support for EPOLL native impl on Linux (disable by
providing -Ddrill.exec.disable_linux_epoll=false) - Move BitData connection
to Multiplex with DataTunnel lev
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/main/java/io/netty/buffer/PooledHeapBufferL.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/main/java/io/netty/buffer/PooledHeapBufferL.java b/exec/bufferl/src/main/java/io/netty/buffer/PooledHeapBufferL.java
deleted file mode 100644
index be7c13b..0000000
--- a/exec/bufferl/src/main/java/io/netty/buffer/PooledHeapBufferL.java
+++ /dev/null
@@ -1,299 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-/*
- * Copyright 2012 The Netty Project
- *
- * The Netty Project licenses this file tothe License at:
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-
-
-import io.netty.util.Recycler;
-import io.netty.util.internal.PlatformDependent;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.ClosedChannelException;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.ScatteringByteChannel;
-
-final class PooledHeapByteBufL extends PooledByteBufL<byte[]> {
-
- private static final Recycler<PooledHeapByteBufL> RECYCLER = new Recycler<PooledHeapByteBufL>() {
- @Override
- protected PooledHeapByteBufL newObject(Handle handle) {
- return new PooledHeapByteBufL(handle, 0);
- }
- };
-
- static PooledHeapByteBufL newInstance(int maxCapacity) {
- PooledHeapByteBufL buf = RECYCLER.get();
- buf.setRefCnt(1);
- buf.maxCapacity(maxCapacity);
- return buf;
- }
-
- private PooledHeapByteBufL(Recycler.Handle recyclerHandle, int maxCapacity) {
- super(recyclerHandle, maxCapacity);
- }
-
- @Override
- public boolean isDirect() {
- return false;
- }
-
- @Override
- protected byte _getByte(int index) {
- return memory[idx(index)];
- }
-
- @Override
- protected short _getShort(int index) {
- index = idx(index);
- return (short) (memory[index] << 8 | memory[index + 1] & 0xFF);
- }
-
- @Override
- protected int _getUnsignedMedium(int index) {
- index = idx(index);
- return (memory[index] & 0xff) << 16 |
- (memory[index + 1] & 0xff) << 8 |
- memory[index + 2] & 0xff;
- }
-
- @Override
- protected int _getInt(int index) {
- index = idx(index);
- return (memory[index] & 0xff) << 24 |
- (memory[index + 1] & 0xff) << 16 |
- (memory[index + 2] & 0xff) << 8 |
- memory[index + 3] & 0xff;
- }
-
- @Override
- protected long _getLong(int index) {
- index = idx(index);
- return ((long) memory[index] & 0xff) << 56 |
- ((long) memory[index + 1] & 0xff) << 48 |
- ((long) memory[index + 2] & 0xff) << 40 |
- ((long) memory[index + 3] & 0xff) << 32 |
- ((long) memory[index + 4] & 0xff) << 24 |
- ((long) memory[index + 5] & 0xff) << 16 |
- ((long) memory[index + 6] & 0xff) << 8 |
- (long) memory[index + 7] & 0xff;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
- checkDstIndex(index, length, dstIndex, dst.capacity());
- if (dst.hasMemoryAddress()) {
- PlatformDependent.copyMemory(memory, idx(index), dst.memoryAddress() + dstIndex, length);
- } else if (dst.hasArray()) {
- getBytes(index, dst.array(), dst.arrayOffset() + dstIndex, length);
- } else {
- dst.setBytes(dstIndex, memory, idx(index), length);
- }
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
- checkDstIndex(index, length, dstIndex, dst.length);
- System.arraycopy(memory, idx(index), dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuffer dst) {
- checkIndex(index);
- dst.put(memory, idx(index), Math.min(capacity() - index, dst.remaining()));
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
- checkIndex(index, length);
- out.write(memory, idx(index), length);
- return this;
- }
-
- @Override
- public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
- checkIndex(index, length);
- index = idx(index);
- return out.write((ByteBuffer) internalNioBuffer().clear().position(index).limit(index + length));
- }
-
- @Override
- protected void _setByte(int index, int value) {
- memory[idx(index)] = (byte) value;
- }
-
- @Override
- protected void _setShort(int index, int value) {
- index = idx(index);
- memory[index] = (byte) (value >>> 8);
- memory[index + 1] = (byte) value;
- }
-
- @Override
- protected void _setMedium(int index, int value) {
- index = idx(index);
- memory[index] = (byte) (value >>> 16);
- memory[index + 1] = (byte) (value >>> 8);
- memory[index + 2] = (byte) value;
- }
-
- @Override
- protected void _setInt(int index, int value) {
- index = idx(index);
- memory[index] = (byte) (value >>> 24);
- memory[index + 1] = (byte) (value >>> 16);
- memory[index + 2] = (byte) (value >>> 8);
- memory[index + 3] = (byte) value;
- }
-
- @Override
- protected void _setLong(int index, long value) {
- index = idx(index);
- memory[index] = (byte) (value >>> 56);
- memory[index + 1] = (byte) (value >>> 48);
- memory[index + 2] = (byte) (value >>> 40);
- memory[index + 3] = (byte) (value >>> 32);
- memory[index + 4] = (byte) (value >>> 24);
- memory[index + 5] = (byte) (value >>> 16);
- memory[index + 6] = (byte) (value >>> 8);
- memory[index + 7] = (byte) value;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
- checkSrcIndex(index, length, srcIndex, src.capacity());
- if (src.hasMemoryAddress()) {
- PlatformDependent.copyMemory(src.memoryAddress() + srcIndex, memory, idx(index), length);
- } else if (src.hasArray()) {
- setBytes(index, src.array(), src.arrayOffset() + srcIndex, length);
- } else {
- src.getBytes(srcIndex, memory, idx(index), length);
- }
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
- checkSrcIndex(index, length, srcIndex, src.length);
- System.arraycopy(src, srcIndex, memory, idx(index), length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuffer src) {
- int length = src.remaining();
- checkIndex(index, length);
- src.get(memory, idx(index), length);
- return this;
- }
-
- @Override
- public int setBytes(int index, InputStream in, int length) throws IOException {
- checkIndex(index, length);
- return in.read(memory, idx(index), length);
- }
-
- @Override
- public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
- checkIndex(index, length);
- index = idx(index);
- try {
- return in.read((ByteBuffer) internalNioBuffer().clear().position(index).limit(index + length));
- } catch (ClosedChannelException e) {
- return -1;
- }
- }
-
- @Override
- public ByteBuf copy(int index, int length) {
- checkIndex(index, length);
- ByteBuf copy = alloc().heapBuffer(length, maxCapacity());
- copy.writeBytes(memory, idx(index), length);
- return copy;
- }
-
- @Override
- public int nioBufferCount() {
- return 1;
- }
-
- @Override
- public ByteBuffer[] nioBuffers(int index, int length) {
- return new ByteBuffer[] { nioBuffer(index, length) };
- }
-
- @Override
- public ByteBuffer internalNioBuffer(int index, int length) {
- checkIndex(index, length);
- index = idx(index);
- return (ByteBuffer) internalNioBuffer().clear().position(index).limit(index + length);
- }
-
- @Override
- public boolean hasArray() {
- return true;
- }
-
- @Override
- public byte[] array() {
- return memory;
- }
-
- @Override
- public int arrayOffset() {
- return offset;
- }
-
- @Override
- public boolean hasMemoryAddress() {
- return false;
- }
-
- @Override
- public long memoryAddress() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- protected ByteBuffer newInternalNioBuffer(byte[] memory) {
- return ByteBuffer.wrap(memory);
- }
-
- @Override
- protected Recycler<?> recycler() {
- return RECYCLER;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/main/java/io/netty/buffer/PooledUnsafeDirectByteBufL.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/main/java/io/netty/buffer/PooledUnsafeDirectByteBufL.java b/exec/bufferl/src/main/java/io/netty/buffer/PooledUnsafeDirectByteBufL.java
deleted file mode 100644
index 8e6d955..0000000
--- a/exec/bufferl/src/main/java/io/netty/buffer/PooledUnsafeDirectByteBufL.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.util.Recycler;
-import io.netty.util.internal.PlatformDependent;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.ClosedChannelException;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.ScatteringByteChannel;
-
-/**
-* A buffer allocated from the netty pool allocator, optimized for little endian access.
-*
-*/
-public final class PooledUnsafeDirectByteBufL extends PooledByteBufL<ByteBuffer> {
-
- private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
-
- private static final Recycler<PooledUnsafeDirectByteBufL> RECYCLER = new Recycler<PooledUnsafeDirectByteBufL>() {
- @Override
- protected PooledUnsafeDirectByteBufL newObject(Handle handle) {
- return new PooledUnsafeDirectByteBufL(handle, 0);
- }
- };
-
- static PooledUnsafeDirectByteBufL newInstance(int maxCapacity) {
- PooledUnsafeDirectByteBufL buf = RECYCLER.get();
- buf.setRefCnt(1);
- buf.maxCapacity(maxCapacity);
- return buf;
- }
-
- private long memoryAddress;
-
- private PooledUnsafeDirectByteBufL(Recycler.Handle recyclerHandle, int maxCapacity) {
- super(recyclerHandle, maxCapacity);
- }
-
- @Override
- void init(PoolChunkL<ByteBuffer> chunk, long handle, int offset, int length, int maxLength) {
- super.init(chunk, handle, offset, length, maxLength);
- initMemoryAddress();
- }
-
- @Override
- void initUnpooled(PoolChunkL<ByteBuffer> chunk, int length) {
- super.initUnpooled(chunk, length);
- initMemoryAddress();
- }
-
- private void initMemoryAddress() {
- memoryAddress = PlatformDependent.directBufferAddress(memory) + offset;
- }
-
- @Override
- protected ByteBuffer newInternalNioBuffer(ByteBuffer memory) {
- return memory.duplicate();
- }
-
- @Override
- public boolean isDirect() {
- return true;
- }
-
- @Override
- protected byte _getByte(int index) {
- return PlatformDependent.getByte(addr(index));
- }
-
- @Override
- protected short _getShort(int index) {
- short v = PlatformDependent.getShort(addr(index));
- return NATIVE_ORDER? v : Short.reverseBytes(v);
- }
-
- @Override
- protected int _getUnsignedMedium(int index) {
- long addr = addr(index);
- return (PlatformDependent.getByte(addr) & 0xff) << 16 |
- (PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
- PlatformDependent.getByte(addr + 2) & 0xff;
- }
-
- @Override
- protected int _getInt(int index) {
- int v = PlatformDependent.getInt(addr(index));
- return NATIVE_ORDER? v : Integer.reverseBytes(v);
- }
-
- @Override
- protected long _getLong(int index) {
- long v = PlatformDependent.getLong(addr(index));
- return NATIVE_ORDER? v : Long.reverseBytes(v);
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
- checkIndex(index, length);
- if (dst == null) {
- throw new NullPointerException("dst");
- }
- if (dstIndex < 0 || dstIndex > dst.capacity() - length) {
- throw new IndexOutOfBoundsException("dstIndex: " + dstIndex);
- }
-
- if (length != 0) {
- if (dst.hasMemoryAddress()) {
- PlatformDependent.copyMemory(addr(index), dst.memoryAddress() + dstIndex, length);
- } else if (dst.hasArray()) {
- PlatformDependent.copyMemory(addr(index), dst.array(), dst.arrayOffset() + dstIndex, length);
- } else {
- dst.setBytes(dstIndex, this, index, length);
- }
- }
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
- checkIndex(index, length);
- if (dst == null) {
- throw new NullPointerException("dst");
- }
- if (dstIndex < 0 || dstIndex > dst.length - length) {
- throw new IndexOutOfBoundsException("dstIndex: " + dstIndex);
- }
- if (length != 0) {
- PlatformDependent.copyMemory(addr(index), dst, dstIndex, length);
- }
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuffer dst) {
- checkIndex(index);
- int bytesToCopy = Math.min(capacity() - index, dst.remaining());
- ByteBuffer tmpBuf = internalNioBuffer();
- index = idx(index);
- tmpBuf.clear().position(index).limit(index + bytesToCopy);
- dst.put(tmpBuf);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
- checkIndex(index, length);
- if (length != 0) {
- byte[] tmp = new byte[length];
- PlatformDependent.copyMemory(addr(index), tmp, 0, length);
- out.write(tmp);
- }
- return this;
- }
-
- @Override
- public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
- checkIndex(index, length);
- if (length == 0) {
- return 0;
- }
-
- ByteBuffer tmpBuf = internalNioBuffer();
- index = idx(index);
- tmpBuf.clear().position(index).limit(index + length);
- return out.write(tmpBuf);
- }
-
- @Override
- protected void _setByte(int index, int value) {
- PlatformDependent.putByte(addr(index), (byte) value);
- }
-
- @Override
- protected void _setShort(int index, int value) {
- PlatformDependent.putShort(addr(index), NATIVE_ORDER ? (short) value : Short.reverseBytes((short) value));
- }
-
- @Override
- protected void _setMedium(int index, int value) {
- long addr = addr(index);
- PlatformDependent.putByte(addr, (byte) (value >>> 16));
- PlatformDependent.putByte(addr + 1, (byte) (value >>> 8));
- PlatformDependent.putByte(addr + 2, (byte) value);
- }
-
- @Override
- protected void _setInt(int index, int value) {
- PlatformDependent.putInt(addr(index), NATIVE_ORDER ? value : Integer.reverseBytes(value));
- }
-
- @Override
- protected void _setLong(int index, long value) {
- PlatformDependent.putLong(addr(index), NATIVE_ORDER ? value : Long.reverseBytes(value));
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
- checkIndex(index, length);
- if (src == null) {
- throw new NullPointerException("src");
- }
- if (srcIndex < 0 || srcIndex > src.capacity() - length) {
- throw new IndexOutOfBoundsException("srcIndex: " + srcIndex);
- }
-
- if (length != 0) {
- if (src.hasMemoryAddress()) {
- PlatformDependent.copyMemory(src.memoryAddress() + srcIndex, addr(index), length);
- } else if (src.hasArray()) {
- PlatformDependent.copyMemory(src.array(), src.arrayOffset() + srcIndex, addr(index), length);
- } else {
- src.getBytes(srcIndex, this, index, length);
- }
- }
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
- checkIndex(index, length);
- if (length != 0) {
- PlatformDependent.copyMemory(src, srcIndex, addr(index), length);
- }
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuffer src) {
- checkIndex(index);
- ByteBuffer tmpBuf = internalNioBuffer();
- if (src == tmpBuf) {
- src = src.duplicate();
- }
-
- index = idx(index);
- tmpBuf.clear().position(index).limit(index + src.remaining());
- tmpBuf.put(src);
- return this;
- }
-
- @Override
- public int setBytes(int index, InputStream in, int length) throws IOException {
- checkIndex(index, length);
- byte[] tmp = new byte[length];
- int readBytes = in.read(tmp);
- if (readBytes > 0) {
- PlatformDependent.copyMemory(tmp, 0, addr(index), readBytes);
- }
- return readBytes;
- }
-
- @Override
- public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
- checkIndex(index, length);
- ByteBuffer tmpNioBuf = internalNioBuffer();
- index = idx(index);
- tmpNioBuf.clear().position(index).limit(index + length);
- try {
- return in.read(tmpNioBuf);
- } catch (ClosedChannelException e) {
- return -1;
- }
- }
-
- @Override
- public ByteBuf copy(int index, int length) {
- checkIndex(index, length);
- PooledUnsafeDirectByteBufL copy = (PooledUnsafeDirectByteBufL) alloc().directBuffer(length, maxCapacity());
- if (length != 0) {
- PlatformDependent.copyMemory(addr(index), copy.addr(0), length);
- copy.setIndex(0, length);
- }
- return copy;
- }
-
- @Override
- public int nioBufferCount() {
- return 1;
- }
-
- @Override
- public ByteBuffer[] nioBuffers(int index, int length) {
- return new ByteBuffer[] { nioBuffer(index, length) };
- }
-
- @Override
- public ByteBuffer internalNioBuffer(int index, int length) {
- checkIndex(index, length);
- index = idx(index);
- return (ByteBuffer) internalNioBuffer().clear().position(index).limit(index + length);
- }
-
- @Override
- public boolean hasArray() {
- return false;
- }
-
- @Override
- public byte[] array() {
- throw new UnsupportedOperationException("direct buffer");
- }
-
- @Override
- public int arrayOffset() {
- throw new UnsupportedOperationException("direct buffer");
- }
-
- @Override
- public boolean hasMemoryAddress() {
- return true;
- }
-
- @Override
- public long memoryAddress() {
- return memoryAddress;
- }
-
- private long addr(int index) {
- return memoryAddress + index;
- }
-
- @Override
- protected Recycler<?> recycler() {
- return RECYCLER;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/main/java/io/netty/buffer/package-info.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/main/java/io/netty/buffer/package-info.java b/exec/bufferl/src/main/java/io/netty/buffer/package-info.java
deleted file mode 100644
index da96959..0000000
--- a/exec/bufferl/src/main/java/io/netty/buffer/package-info.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-
-/**
-* This package implements a memory manager for off-heap data based on pools and arenas.
-* It is derived from the netty memory manager with the following changes:
-* - Classes have been redefined to store data little-endian. (Append "L" to class names)
-* - A "trim" method has been added to reduce a buffer's maxCapacity and free the extra memory.
-* - allocate(min, max)
-*
-* This manager is based on the following memory abstractions:
-* - a "chunk" is a piece of memory allocated from the operating system,
-* - a "page" is is a piece of memory allocated from a chunk, and
-* - an "element" is a piece of memory allocated from a page.
-*
-* This memory manager classifies the memory request according to the size of requested memory.
-* - "subpage" memory manager which breaks a page into equal sized elements.
-* It uses a bitmap to indicate which ones are free. (why not linked list?)
-* - "normal" memory manager which breaks a chunk into runs of pages.
-* Management is done using the "buddy system", so each run is a power-of-2 pages.
-* - "huge" memory manager which allocates memory larger than a chunk.
-* It goes straight to the operating system.
-*
-* The PoolArena utilizes these three memory managers, trying to minimize fragmentation
-* in the big buffers while keeping good performance for the small buffers.
-*
-* The three submanagers are:
-* subpage allocations. A page is divided into equal sized elements with a bitmask indicating
-* which elements are free. The page is placed into a pool (linked list)
-* according to the size of the elements.
-* Normal allocations. A chunk is divided into multiple pages according to the buddy system.
-* Chunks are kept in a list partially ordered to minimize fragmentation.
-* Huge allocations. Memory larger than a chunk is allocated directly from the OS.
-*
-* To avoid contention, there are multiple "arenas" rather than a single monolithic memory manager.
-* Threads are assigned to an arena on a round-robin basis, and a buffer is always returned
-* to the arena of its origin.
-*
-*
-* Note for "normal" allocations: Rather than keeping one fully ordered list of chunks,
-* it keeps several lists, and always
-* searches the first list, followed by the next list. Thus it achieves an approximate ordering without
-* having to do a full sort.
-*
-*
-* This PoolArena allocator is configured according to Java environment variables.
-* io.netty.allocator.numDirectArenas - the number of arenas to divide among threads.
-* io.netty.allocator.pageSize - The size of each page, must be power of 2.
-* io.netty.allocator.maxOrder - Specifies chunkSize as pageSize * 2^maxOrder.
-*
-* It may not make sense to use this allocator for small allocations.
-* Each buffer which is allocated uses a Java class which must also be allocated.
-* Thus we may not save on Java memory management for small buffers.
-*/
-
-package io.netty.buffer;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/test/java/io/netty/buffer/TestPoolChunkTrim.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/test/java/io/netty/buffer/TestPoolChunkTrim.java b/exec/bufferl/src/test/java/io/netty/buffer/TestPoolChunkTrim.java
deleted file mode 100644
index f7ba7f2..0000000
--- a/exec/bufferl/src/test/java/io/netty/buffer/TestPoolChunkTrim.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import io.netty.util.internal.StringUtil;
-
-import java.nio.ByteBuffer;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestPoolChunkTrim {
-
- int pageSize=8192; // default
-
- /** A convenience method to do all the tests. */
- public void test() {
- singlePageTest();
- tiniestTinyTest();
- largestTinyTest();
- smallestSmallTest();
- largestSmallTest();
- trimTest();
- trimTiny();
- tinyGrow();
- }
-
- /**
- * Unit test the memory allocator and trim() function.
- * The results are confirmed by examining the state of the memory allocator.
- * In this test, we are working with a single chunk and a
- * small set of subpage allocations.
- *
- * Chunk status can be verified by matching with
- * "Chunk ... <bytes consumed>/".
- * The subpage allocators are verified by matching with
- * "<nr allocated>/ ... elemSize: <size>"
- * *
- * A cleaner approach would create new methods to query
- * - how many pages (total) have been allocated, and
- * - how many elements of a particular size have been allocated.
- */
-
- /** Allocate and free a single page */
- @Test
- public void singlePageTest() {
- normalTest(pageSize/2+1, pageSize, 1);
- }
-
- /** Allocate and free a larger run of pages */
- @Test
- public void multiPageTest() {
- normalTest(pageSize*25+1, pageSize*31+1, 32);
- }
-
- /** Allocate and free the tiniest tiny allocation */
- @Test
- public void tiniestTinyTest() {
- subpageTest(1, 1, 16);
- }
-
- /** Allocate and free the largest tiny allocation */
- @Test
- public void largestTinyTest() {
- subpageTest(512-16-15, 512-16-15, 512-16);
- }
-
- /** Allocate and free the smallest small allocation */
- @Test
- public void smallestSmallTest() {
- subpageTest(512-15, 512-15, 512);
- }
-
- /** Allocate and free the largest small allocation */
- @Test
- public void largestSmallTest() {
- subpageTest(pageSize/2-1, pageSize/2-1, pageSize/2);
- }
-
-
- /** Trim a large block to a smaller block. */
- @Test
- public void trimTest() {
-
- // Allocate a large block and trim to a single page
- TestAllocator allocator = new TestAllocator();
- ByteBuf block = allocator.directBuffer(25*pageSize, 256*pageSize);
- Assert.assertTrue(25*pageSize <= block.capacity() && block.capacity() <= 256*pageSize);
- allocator.assertPages(256);
-
- block.capacity(pageSize/2+1);
- Assert.assertTrue(block.capacity() == pageSize/2+1);
- allocator.assertPages(1);
- }
-
-
- /** Trim a page down to a tiny buffer. */
- @Test
- public void trimTiny() {
-
- // Allocate a single page
- TestAllocator allocator = new TestAllocator();
- ByteBuf block = allocator.directBuffer(pageSize, pageSize);
- Assert.assertTrue(pageSize == block.capacity());
- allocator.assertPages(1);
-
- // Trim the single page to a tiny size.
- block.capacity(31);
- allocator.assertElement(1, 32).assertPages(1);
- }
-
-
- /** Grow a tiny buffer to a normal one */
- @Test
- public void tinyGrow() {
-
- // Allocate a tiny buffer
- TestAllocator allocator = new TestAllocator();
- ByteBuf block = allocator.directBuffer(1, 1);
- allocator.assertPages(1).assertElement(1, 16);
-
- // Resize the tiny block to two pages
- block = block.capacity(pageSize+1);
- Assert.assertTrue(block.capacity() == pageSize+1);
- allocator.assertPages(3).assertElement(0, 16);
- }
-
-
-
-
-
- /** Test the allocation and free of a "normal" allocation */
- private void normalTest(int min, int max, int pages) {
- TestAllocator allocator = new TestAllocator();
-
- ByteBuf block = allocator.directBuffer(min, max);
- Assert.assertTrue(block.capacity() >= min);
- Assert.assertTrue(block.capacity() <= max);
- allocator.assertPages(pages);
-
- block.release();
- allocator.assertPages(0);
- }
-
-
-
- /** Test the allocation and free of a "subpage" allocation */
- private void subpageTest(int min, int max, int expected) {
-
- TestAllocator allocator = new TestAllocator();
-
- // Allocate the buffer and verify we have the expected number of pages
- ByteBuf block = allocator.directBuffer(min, max);
- Assert.assertTrue(block.capacity() >= min);
- Assert.assertTrue(block.capacity() <= max);
- allocator.assertPages(1).assertElement(1, expected);
-
- // Release the buffer. Verify the element is returned to pool and page still allocated.
- block.release();
- allocator.assertPages(1).assertElement(0, expected);
- }
-
-
-
- /** An allocator with some stuff added to aid testing */
- class TestAllocator extends PooledByteBufAllocatorL {
-
- TestAllocator() {super(true);}
-
- public TestAllocator assertPages(int pages) {
- return assertMatch("Chunk.* "+pages*pageSize+"/");
- }
-
- public TestAllocator assertElement(int count, int size) {
- return assertMatch(count+"/.*elemSize: "+size);
- }
-
- /**
- * Verify our current state matches the pattern.
- *
- * Note: Uses the existing "toString()" method and extracts information
- * by matching a pattern to one of the output lines.
- *
- */
- TestAllocator assertMatch(String pattern) {
-
- // Get our current state as a string
- String s = toString();
-
- // Do for each line in the string
- for (int f=0, l=s.indexOf(StringUtil.NEWLINE, f); l != -1; f=l+StringUtil.NEWLINE.length(), l=s.indexOf(StringUtil.NEWLINE, f)) {
-
- // if the line contains pattern, then success.
- if (s.substring(f,l).matches(".*"+pattern+".*")) return this;
- }
-
- // We didn't find a matching line, so fail the test
- Assert.fail("Test failed to match pattern " + pattern);
- return this;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 81104e1..7a8598a 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -90,6 +90,12 @@
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>2.2.0</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ <classifier>linux-x86_64</classifier>
+ <version>4.0.20.Final</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.ext</groupId>
@@ -166,11 +172,11 @@
<artifactId>drill-common</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.drill.exec</groupId>
- <version>${project.version}</version>
- <artifactId>drill-buffers</artifactId>
- </dependency>
+<!-- <dependency> -->
+<!-- <groupId>org.apache.drill.exec</groupId> -->
+<!-- <version>${project.version}</version> -->
+<!-- <artifactId>drill-buffers</artifactId> -->
+<!-- </dependency> -->
<dependency>
<groupId>org.apache.drill</groupId>
<artifactId>drill-common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
index 872c0b8..3afa20c 100644
--- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -17,14 +17,12 @@ import com.google.common.collect.ObjectArrays;
import com.google.common.base.Charsets;
import com.google.common.collect.ObjectArrays;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
+import io.netty.buffer.*;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.drill.exec.expr.fn.impl.StringFunctionUtil;
-import org.apache.drill.exec.memory.AccountingByteBuf;
-import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.*;
import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.*;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
index d9715a7..406d883 100644
--- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
@@ -16,15 +16,6 @@
* limitations under the License.
*/
-import java.lang.Long;
-import java.lang.Override;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-
-import org.joda.time.Period;
-
<@pp.dropOutputFile />
<#list vv.types as type>
<#list type.minor as minor>
@@ -790,11 +781,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
} else if (allocationMonitor > 0) {
allocationMonitor = 0;
}
- data.writerIndex(idx);
- if (data instanceof AccountingByteBuf) {
- data.capacity(idx);
- data.writerIndex(idx);
- }
+ VectorTrimmer.trim(data, idx);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
index 8535f99..7f01058 100644
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -497,10 +497,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
} else if (allocationMonitor > 0) {
allocationMonitor = 0;
}
- if (data instanceof AccountingByteBuf) {
- data.capacity(idx);
- data.writerIndex(idx);
- }
+ VectorTrimmer.trim(data, idx);
offsetVector.getMutator().setValueCount(valueCount+1);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/io/netty/buffer/AccountingByteBuf.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/AccountingByteBuf.java b/exec/java-exec/src/main/java/io/netty/buffer/AccountingByteBuf.java
new file mode 100644
index 0000000..e5b0f94
--- /dev/null
+++ b/exec/java-exec/src/main/java/io/netty/buffer/AccountingByteBuf.java
@@ -0,0 +1,866 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.ScatteringByteChannel;
+import java.nio.charset.Charset;
+
+import org.apache.drill.exec.memory.Accountor;
+
+public class AccountingByteBuf extends ByteBuf{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AccountingByteBuf.class);
+
+ private final UnsafeDirectLittleEndian b;
+ private volatile Accountor acct;
+ private volatile int size;
+
+ public AccountingByteBuf(Accountor a, ByteBuf b) {
+ super();
+ this.b = (UnsafeDirectLittleEndian) b;
+ this.acct = a;
+ this.size = b.capacity();
+ }
+
+ @Override
+ public int refCnt() {
+ return b.refCnt();
+ }
+
+ public boolean transferAccounting(Accountor target){
+ boolean outcome = acct.transferTo(target, this, size);
+ acct = target;
+ return outcome;
+ }
+
+ @Override
+ public synchronized boolean release() {
+ if(b.release()){
+ acct.release(this, size);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized boolean release(int decrement) {
+ if(b.release(decrement)){
+ acct.release(this, size);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public int capacity() {
+ return b.capacity();
+ }
+
+ @Override
+ public synchronized ByteBuf capacity(int newCapacity) {
+ if(newCapacity == size){
+ return this;
+ }else if(newCapacity < size){
+ b.capacity(newCapacity);
+ int diff = size - b.capacity();
+ acct.releasePartial(this, diff);
+ this.size = size - diff;
+ return this;
+ }else{
+ throw new UnsupportedOperationException("Accounting byte buf doesn't support increasing allocations.");
+ }
+
+ }
+
+ @Override
+ public int maxCapacity() {
+ return size;
+ }
+
+ @Override
+ public ByteBufAllocator alloc() {
+ return b.alloc();
+ }
+
+ @Override
+ public ByteOrder order() {
+ return ByteOrder.LITTLE_ENDIAN;
+ }
+
+ @Override
+ public ByteBuf order(ByteOrder endianness) {
+// if(endianness != ByteOrder.LITTLE_ENDIAN) throw new UnsupportedOperationException("Drill buffers only support little endian.");
+ return this;
+ }
+
+ @Override
+ public ByteBuf unwrap() {
+ return this;
+ }
+
+ @Override
+ public boolean isDirect() {
+ return true;
+ }
+
+ @Override
+ public int readerIndex() {
+ return b.readerIndex();
+ }
+
+ @Override
+ public ByteBuf readerIndex(int readerIndex) {
+ b.readerIndex(readerIndex);
+ return this;
+ }
+
+ @Override
+ public int writerIndex() {
+ return b.writerIndex();
+ }
+
+ @Override
+ public ByteBuf writerIndex(int writerIndex) {
+ b.writerIndex(writerIndex);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setIndex(int readerIndex, int writerIndex) {
+ b.setIndex(readerIndex, writerIndex);
+ return this;
+ }
+
+ @Override
+ public int readableBytes() {
+ return b.readableBytes();
+ }
+
+ @Override
+ public int writableBytes() {
+ return b.writableBytes();
+ }
+
+ @Override
+ public int maxWritableBytes() {
+ return b.maxWritableBytes();
+ }
+
+ @Override
+ public boolean isReadable() {
+ return b.isReadable();
+ }
+
+ @Override
+ public boolean isReadable(int size) {
+ return b.isReadable(size);
+ }
+
+ @Override
+ public boolean isWritable() {
+ return b.isWritable();
+ }
+
+ @Override
+ public boolean isWritable(int size) {
+ return b.isWritable(size);
+ }
+
+ @Override
+ public ByteBuf clear() {
+ b.clear();
+ return this;
+ }
+
+ @Override
+ public ByteBuf markReaderIndex() {
+ b.markReaderIndex();
+ return this;
+ }
+
+ @Override
+ public ByteBuf resetReaderIndex() {
+ b.resetReaderIndex();
+ return this;
+ }
+
+ @Override
+ public ByteBuf markWriterIndex() {
+ b.markWriterIndex();
+ return this;
+ }
+
+ @Override
+ public ByteBuf resetWriterIndex() {
+ b.resetWriterIndex();
+ return this;
+ }
+
+ @Override
+ public ByteBuf discardReadBytes() {
+ b.discardReadBytes();
+ return this;
+ }
+
+ @Override
+ public ByteBuf discardSomeReadBytes() {
+ b.discardSomeReadBytes();
+ return this;
+ }
+
+ @Override
+ public ByteBuf ensureWritable(int minWritableBytes) {
+ b.ensureWritable(minWritableBytes);
+ return this;
+ }
+
+ @Override
+ public int ensureWritable(int minWritableBytes, boolean force) {
+ return b.ensureWritable(minWritableBytes, false);
+ }
+
+ @Override
+ public boolean getBoolean(int index) {
+ return b.getBoolean(index);
+ }
+
+ @Override
+ public byte getByte(int index) {
+ return b.getByte(index);
+ }
+
+ @Override
+ public short getUnsignedByte(int index) {
+ return b.getUnsignedByte(index);
+ }
+
+ @Override
+ public short getShort(int index) {
+ return b.getShort(index);
+ }
+
+ @Override
+ public int getUnsignedShort(int index) {
+ return b.getUnsignedShort(index);
+ }
+
+ @Override
+ public int getMedium(int index) {
+ return b.getMedium(index);
+ }
+
+ @Override
+ public int getUnsignedMedium(int index) {
+ return b.getUnsignedMedium(index);
+ }
+
+ @Override
+ public int getInt(int index) {
+ return b.getInt(index);
+ }
+
+ @Override
+ public long getUnsignedInt(int index) {
+ return b.getUnsignedInt(index);
+ }
+
+ @Override
+ public long getLong(int index) {
+ return b.getLong(index);
+ }
+
+ @Override
+ public char getChar(int index) {
+ return b.getChar(index);
+ }
+
+ @Override
+ public float getFloat(int index) {
+ return b.getFloat(index);
+ }
+
+ @Override
+ public double getDouble(int index) {
+ return b.getDouble(index);
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuf dst) {
+ b.getBytes(index, dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuf dst, int length) {
+ b.getBytes(index, dst, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
+ b.getBytes(index, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, byte[] dst) {
+ b.getBytes(index, dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
+ b.getBytes(index, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuffer dst) {
+ b.getBytes(index, dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
+ b.getBytes(index, out, length);
+ return this;
+ }
+
+ @Override
+ public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
+ return b.getBytes(index, out, length);
+ }
+
+ @Override
+ public ByteBuf setBoolean(int index, boolean value) {
+ b.setBoolean(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setByte(int index, int value) {
+ b.setByte(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setShort(int index, int value) {
+ b.setShort(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setMedium(int index, int value) {
+ b.setMedium(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setInt(int index, int value) {
+ b.setInt(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setLong(int index, long value) {
+ b.setLong(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setChar(int index, int value) {
+ b.setChar(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setFloat(int index, float value) {
+ b.setFloat(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setDouble(int index, double value) {
+ b.setDouble(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuf src) {
+ b.setBytes(index, src);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuf src, int length) {
+ b.setBytes(index, src, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
+ b.setBytes(index, src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, byte[] src) {
+ b.setBytes(index, src);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
+ b.setBytes(index, src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuffer src) {
+ b.setBytes(index, src);
+ return this;
+ }
+
+ @Override
+ public int setBytes(int index, InputStream in, int length) throws IOException {
+ return b.setBytes(index, in, length);
+ }
+
+ @Override
+ public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
+ return b.setBytes(index, in, length);
+ }
+
+ @Override
+ public ByteBuf setZero(int index, int length) {
+ b.setZero(index, length);
+ return this;
+ }
+
+ @Override
+ public boolean readBoolean() {
+ return b.readBoolean();
+ }
+
+ @Override
+ public byte readByte() {
+ return b.readByte();
+ }
+
+ @Override
+ public short readUnsignedByte() {
+ return b.readUnsignedByte();
+ }
+
+ @Override
+ public short readShort() {
+ return b.readShort();
+ }
+
+ @Override
+ public int readUnsignedShort() {
+ return b.readUnsignedShort();
+ }
+
+ @Override
+ public int readMedium() {
+ return b.readMedium();
+ }
+
+ @Override
+ public int readUnsignedMedium() {
+ return b.readUnsignedMedium();
+ }
+
+ @Override
+ public int readInt() {
+ return b.readInt();
+ }
+
+ @Override
+ public long readUnsignedInt() {
+ return b.readUnsignedInt();
+ }
+
+ @Override
+ public long readLong() {
+ return b.readLong();
+ }
+
+ @Override
+ public char readChar() {
+ return b.readChar();
+ }
+
+ @Override
+ public float readFloat() {
+ return b.readFloat();
+ }
+
+ @Override
+ public double readDouble() {
+ return b.readDouble();
+ }
+
+ @Override
+ public ByteBuf readBytes(int length) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf readSlice(int length) {
+ ByteBuf slice = slice(readerIndex(), length);
+ readerIndex(readerIndex() + length);
+ return slice;
+ }
+
+
+ @Override
+ public ByteBuf readBytes(ByteBuf dst) {
+ b.readBytes(dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf readBytes(ByteBuf dst, int length) {
+ b.readBytes(dst, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf readBytes(ByteBuf dst, int dstIndex, int length) {
+ b.readBytes(dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf readBytes(byte[] dst) {
+ b.readBytes(dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf readBytes(byte[] dst, int dstIndex, int length) {
+ b.readBytes(dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf readBytes(ByteBuffer dst) {
+ b.readBytes(dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf readBytes(OutputStream out, int length) throws IOException {
+ b.readBytes(out, length);
+ return null;
+ }
+
+ @Override
+ public int readBytes(GatheringByteChannel out, int length) throws IOException {
+ return b.readBytes(out, length);
+ }
+
+ @Override
+ public ByteBuf skipBytes(int length) {
+ b.skipBytes(length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeBoolean(boolean value) {
+ b.writeBoolean(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeByte(int value) {
+ b.writeByte(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeShort(int value) {
+ b.writeShort(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeMedium(int value) {
+ b.writeMedium(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeInt(int value) {
+ b.writeInt(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeLong(long value) {
+ b.writeLong(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeChar(int value) {
+ b.writeChar(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeFloat(float value) {
+ b.writeFloat(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeDouble(double value) {
+ b.writeDouble(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeBytes(ByteBuf src) {
+ b.writeBytes(src);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeBytes(ByteBuf src, int length) {
+ b.writeBytes(src, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeBytes(ByteBuf src, int srcIndex, int length) {
+ b.writeBytes(src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeBytes(byte[] src) {
+ b.writeBytes(src);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeBytes(byte[] src, int srcIndex, int length) {
+ b.writeBytes(src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeBytes(ByteBuffer src) {
+ b.writeBytes(src);
+ return this;
+ }
+
+ @Override
+ public int writeBytes(InputStream in, int length) throws IOException {
+ return b.writeBytes(in, length);
+ }
+
+ @Override
+ public int writeBytes(ScatteringByteChannel in, int length) throws IOException {
+ return b.writeBytes(in, length);
+ }
+
+ @Override
+ public ByteBuf writeZero(int length) {
+ b.writeZero(length);
+ return this;
+ }
+
+ @Override
+ public int indexOf(int fromIndex, int toIndex, byte value) {
+ return b.indexOf(fromIndex, toIndex, value);
+ }
+
+ @Override
+ public int bytesBefore(byte value) {
+ return b.bytesBefore(value);
+ }
+
+ @Override
+ public int bytesBefore(int length, byte value) {
+ return b.bytesBefore(length, value);
+ }
+
+ @Override
+ public int bytesBefore(int index, int length, byte value) {
+ return b.bytesBefore(index, length, value);
+ }
+
+ @Override
+ public int forEachByte(ByteBufProcessor processor) {
+ return b.forEachByte(processor);
+ }
+
+ @Override
+ public int forEachByte(int index, int length, ByteBufProcessor processor) {
+ return b.forEachByte(index, length, processor);
+ }
+
+ @Override
+ public int forEachByteDesc(ByteBufProcessor processor) {
+ return b.forEachByteDesc(processor);
+ }
+
+ @Override
+ public int forEachByteDesc(int index, int length, ByteBufProcessor processor) {
+ return b.forEachByteDesc(index, length, processor);
+ }
+
+ @Override
+ public ByteBuf copy() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf copy(int index, int length) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf slice() {
+ return slice(b.readerIndex(), readableBytes());
+ }
+
+ @Override
+ public ByteBuf slice(int index, int length) {
+// if (length == 0) {
+// return Unpooled.EMPTY_BUFFER;
+// }
+
+ return new SlicedByteBuf(this, index, length);
+ }
+
+ @Override
+ public ByteBuf duplicate() {
+ return new DuplicatedByteBuf(this);
+ }
+
+ @Override
+ public int nioBufferCount() {
+ return b.nioBufferCount();
+ }
+
+ @Override
+ public ByteBuffer nioBuffer() {
+ return b.nioBuffer();
+ }
+
+ @Override
+ public ByteBuffer nioBuffer(int index, int length) {
+ return b.nioBuffer(index, length);
+ }
+
+ @Override
+ public ByteBuffer internalNioBuffer(int index, int length) {
+ return b.internalNioBuffer(index, length);
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers() {
+ return b.nioBuffers();
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers(int index, int length) {
+ return b.nioBuffers(index, length);
+ }
+
+ @Override
+ public boolean hasArray() {
+ return b.hasArray();
+ }
+
+ @Override
+ public byte[] array() {
+ return b.array();
+ }
+
+ @Override
+ public int arrayOffset() {
+ return b.arrayOffset();
+ }
+
+ @Override
+ public boolean hasMemoryAddress() {
+ return b.hasMemoryAddress();
+ }
+
+ @Override
+ public long memoryAddress() {
+ return b.memoryAddress();
+ }
+
+ @Override
+ public String toString(Charset charset) {
+ return b.toString(charset);
+ }
+
+ @Override
+ public String toString(int index, int length, Charset charset) {
+ return b.toString(index, length, charset);
+ }
+
+ @Override
+ public int hashCode() {
+ return System.identityHashCode(this);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // identity equals only.
+ return this == obj;
+ }
+
+ @Override
+ public String toString() {
+ return "AccountingByteBuf [Inner buffer=" + b + ", size=" + size + "]";
+ }
+
+ @Override
+ public int compareTo(ByteBuf buffer) {
+ return b.compareTo(buffer);
+ }
+
+
+ @Override
+ public ByteBuf retain(int increment) {
+ b.retain(increment);
+ return this;
+ }
+
+ @Override
+ public ByteBuf retain() {
+ b.retain();
+ return this;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
new file mode 100644
index 0000000..a8482ae
--- /dev/null
+++ b/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import java.nio.ByteBuffer;
+
+public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PooledByteBufAllocatorL.class);
+
+ public static final PooledByteBufAllocatorL DEFAULT = new PooledByteBufAllocatorL();
+
+// public final UnsafeDirectLittleEndian emptyBuf;
+
+ public PooledByteBufAllocatorL() {
+ super(true);
+// emptyBuf = newDirectBuffer(0,0);
+ }
+
+ @Override
+ protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) {
+ throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
+ }
+
+ @Override
+ protected UnsafeDirectLittleEndian newDirectBuffer(int initialCapacity, int maxCapacity) {
+ PoolThreadCache cache = threadCache.get();
+ PoolArena<ByteBuffer> directArena = cache.directArena;
+
+ ByteBuf buf;
+ if (directArena != null) {
+ buf = directArena.allocate(cache, initialCapacity, maxCapacity);
+ } else {
+ throw new UnsupportedOperationException("Drill requries that the allocator operates in DirectBuffer mode.");
+ }
+
+ if(buf instanceof PooledUnsafeDirectByteBuf){
+ return new UnsafeDirectLittleEndian( (PooledUnsafeDirectByteBuf) buf);
+ }else{
+ throw new UnsupportedOperationException("Drill requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality.");
+ }
+
+ }
+
+
+ @Override
+ public ByteBuf directBuffer(int initialCapacity, int maxCapacity) {
+ if (initialCapacity == 0 && maxCapacity == 0) {
+ newDirectBuffer(initialCapacity, maxCapacity);
+ }
+ validate(initialCapacity, maxCapacity);
+ return newDirectBuffer(initialCapacity, maxCapacity);
+ }
+
+ @Override
+ public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
+ throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
+ }
+
+
+ private static void validate(int initialCapacity, int maxCapacity) {
+ if (initialCapacity < 0) {
+ throw new IllegalArgumentException("initialCapacity: " + initialCapacity + " (expectd: 0+)");
+ }
+ if (initialCapacity > maxCapacity) {
+ throw new IllegalArgumentException(String.format(
+ "initialCapacity: %d (expected: not greater than maxCapacity(%d)",
+ initialCapacity, maxCapacity));
+ }
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
new file mode 100644
index 0000000..ad85a63
--- /dev/null
+++ b/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.netty.buffer;
+
+import io.netty.util.internal.PlatformDependent;
+
+import java.nio.ByteOrder;
+
+final class UnsafeDirectLittleEndian extends WrappedByteBuf {
+ private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
+ private final PooledUnsafeDirectByteBuf wrapped;
+ private final long memoryAddress;
+
+ UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf) {
+ super(buf);
+ if(!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) throw new IllegalStateException("Drill only runs on LittleEndian systems.");
+ wrapped = buf;
+ this.memoryAddress = buf.memoryAddress();
+ }
+
+ private long addr(int index) {
+ return memoryAddress + index;
+ }
+
+ @Override
+ public long getLong(int index) {
+ wrapped.checkIndex(index, 8);
+ long v = PlatformDependent.getLong(addr(index));
+ return v;
+ }
+
+ @Override
+ public float getFloat(int index) {
+ return Float.intBitsToFloat(getInt(index));
+ }
+
+ @Override
+ public double getDouble(int index) {
+ return Double.longBitsToDouble(getLong(index));
+ }
+
+ @Override
+ public char getChar(int index) {
+ return (char) getShort(index);
+ }
+
+ @Override
+ public long getUnsignedInt(int index) {
+ return getInt(index) & 0xFFFFFFFFL;
+ }
+
+ @Override
+ public int getInt(int index) {
+ wrapped.checkIndex(index, 4);
+ int v = PlatformDependent.getInt(addr(index));
+ return v;
+ }
+
+ @Override
+ public int getUnsignedShort(int index) {
+ return getShort(index) & 0xFFFF;
+ }
+
+ @Override
+ public short getShort(int index) {
+ wrapped.checkIndex(index, 2);
+ short v = PlatformDependent.getShort(addr(index));
+ return v;
+ }
+
+ @Override
+ public ByteBuf setShort(int index, int value) {
+ wrapped.checkIndex(index, 2);
+ _setShort(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setInt(int index, int value) {
+ wrapped.checkIndex(index, 4);
+ _setInt(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setLong(int index, long value) {
+ wrapped.checkIndex(index, 8);
+ _setLong(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setChar(int index, int value) {
+ setShort(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setFloat(int index, float value) {
+ setInt(index, Float.floatToRawIntBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf setDouble(int index, double value) {
+ setLong(index, Double.doubleToRawLongBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeShort(int value) {
+ wrapped.ensureWritable(2);
+ _setShort(wrapped.writerIndex, value);
+ wrapped.writerIndex += 2;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeInt(int value) {
+ wrapped.ensureWritable(4);
+ _setInt(wrapped.writerIndex, value);
+ wrapped.writerIndex += 4;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeLong(long value) {
+ wrapped.ensureWritable(8);
+ _setLong(wrapped.writerIndex, value);
+ wrapped.writerIndex += 8;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeChar(int value) {
+ writeShort(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeFloat(float value) {
+ writeInt(Float.floatToRawIntBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeDouble(double value) {
+ writeLong(Double.doubleToRawLongBits(value));
+ return this;
+ }
+
+ private void _setShort(int index, int value) {
+ PlatformDependent.putShort(addr(index), (short) value);
+ }
+
+ private void _setInt(int index, int value) {
+ PlatformDependent.putInt(addr(index), value);
+ }
+
+ private void _setLong(int index, long value) {
+ PlatformDependent.putLong(addr(index), value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
index 7d7f2ce..090a760 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
@@ -21,7 +21,6 @@ import static com.google.common.base.Preconditions.checkState;
import static org.apache.drill.exec.proto.UserProtos.QueryResultsMode.STREAM_FULL;
import static org.apache.drill.exec.proto.UserProtos.RunQuery.newBuilder;
import io.netty.buffer.ByteBuf;
-import io.netty.channel.nio.NioEventLoopGroup;
import java.io.Closeable;
import java.io.IOException;
@@ -47,7 +46,7 @@ import org.apache.drill.exec.proto.UserProtos.UserProperties;
import org.apache.drill.exec.rpc.BasicClientWithConnection.ServerConnection;
import org.apache.drill.exec.rpc.ChannelClosedException;
import org.apache.drill.exec.rpc.DrillRpcFuture;
-import org.apache.drill.exec.rpc.NamedThreadFactory;
+import org.apache.drill.exec.rpc.TransportCheck;
import org.apache.drill.exec.rpc.RpcConnectionHandler;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.user.ConnectionThrottle;
@@ -161,7 +160,8 @@ public class DrillClient implements Closeable, ConnectionThrottle{
checkState(!endpoints.isEmpty(), "No DrillbitEndpoint can be found");
// just use the first endpoint for now
DrillbitEndpoint endpoint = endpoints.iterator().next();
- this.client = new UserClient(allocator, new NioEventLoopGroup(config.getInt(ExecConstants.CLIENT_RPC_THREADS), new NamedThreadFactory("Client-")));
+
+ this.client = new UserClient(allocator, TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.CLIENT_RPC_THREADS), "Client-"));
logger.debug("Connecting to server {}:{}", endpoint.getAddress(), endpoint.getUserPort());
connect(endpoint);
connected = true;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
index 83f4262..b4c3f9c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
@@ -148,7 +148,7 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
@Override
public void unregister(RegistrationHandle handle) {
- if (!(handle instanceof ZKRegistrationHandle)) throw new UnsupportedOperationException("Unknown handle type");
+ if (!(handle instanceof ZKRegistrationHandle)) throw new UnsupportedOperationException("Unknown handle type: " + handle.getClass().getName());
ZKRegistrationHandle h = (ZKRegistrationHandle) handle;
try {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountingByteBuf.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountingByteBuf.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountingByteBuf.java
deleted file mode 100644
index f2d695e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountingByteBuf.java
+++ /dev/null
@@ -1,867 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.ByteBufProcessor;
-import io.netty.buffer.DuplicatedByteBuf;
-import io.netty.buffer.PooledUnsafeDirectByteBufL;
-import io.netty.buffer.SlicedByteBuf;
-import io.netty.buffer.SwappedByteBuf;
-import io.netty.buffer.Unpooled;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.ScatteringByteChannel;
-import java.nio.charset.Charset;
-
-public class AccountingByteBuf extends ByteBuf{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AccountingByteBuf.class);
-
- private final PooledUnsafeDirectByteBufL b;
- private final Accountor acct;
- private volatile int size;
-
- public AccountingByteBuf(Accountor a, PooledUnsafeDirectByteBufL b) {
- super();
- this.b = b;
- this.acct = a;
- this.size = b.capacity();
- }
-
- @Override
- public int refCnt() {
- return b.refCnt();
- }
-
- @Override
- public synchronized boolean release() {
- if(b.release()){
- acct.release(this, size);
- return true;
- }
- return false;
- }
-
- @Override
- public synchronized boolean release(int decrement) {
- if(b.release(decrement)){
- acct.release(this, size);
- return true;
- }
- return false;
- }
-
- @Override
- public int capacity() {
- return b.capacity();
- }
-
- @Override
- public synchronized ByteBuf capacity(int newCapacity) {
- if(newCapacity == size){
- return this;
- }else if(newCapacity < size){
- b.capacity(newCapacity);
- int diff = size - b.capacity();
- acct.releasePartial(this, diff);
- this.size = size - diff;
- return this;
- }else{
- throw new UnsupportedOperationException("Accounting byte buf doesn't support increasing allocations.");
- }
-
- }
-
- @Override
- public int maxCapacity() {
- return size;
- }
-
- @Override
- public ByteBufAllocator alloc() {
- return b.alloc();
- }
-
- @Override
- public ByteOrder order() {
- return ByteOrder.LITTLE_ENDIAN;
- }
-
- @Override
- public ByteBuf order(ByteOrder endianness) {
- if(endianness != ByteOrder.BIG_ENDIAN) throw new UnsupportedOperationException("Drill buffers only support big endian.");
- return this;
- }
-
- @Override
- public ByteBuf unwrap() {
- return this;
- }
-
- @Override
- public boolean isDirect() {
- return true;
- }
-
- @Override
- public int readerIndex() {
- return b.readerIndex();
- }
-
- @Override
- public ByteBuf readerIndex(int readerIndex) {
- b.readerIndex(readerIndex);
- return this;
- }
-
- @Override
- public int writerIndex() {
- return b.writerIndex();
- }
-
- @Override
- public ByteBuf writerIndex(int writerIndex) {
- b.writerIndex(writerIndex);
- return this;
- }
-
- @Override
- public ByteBuf setIndex(int readerIndex, int writerIndex) {
- b.setIndex(readerIndex, writerIndex);
- return this;
- }
-
- @Override
- public int readableBytes() {
- return b.readableBytes();
- }
-
- @Override
- public int writableBytes() {
- return b.writableBytes();
- }
-
- @Override
- public int maxWritableBytes() {
- return b.maxWritableBytes();
- }
-
- @Override
- public boolean isReadable() {
- return b.isReadable();
- }
-
- @Override
- public boolean isReadable(int size) {
- return b.isReadable(size);
- }
-
- @Override
- public boolean isWritable() {
- return b.isWritable();
- }
-
- @Override
- public boolean isWritable(int size) {
- return b.isWritable(size);
- }
-
- @Override
- public ByteBuf clear() {
- b.clear();
- return this;
- }
-
- @Override
- public ByteBuf markReaderIndex() {
- b.markReaderIndex();
- return this;
- }
-
- @Override
- public ByteBuf resetReaderIndex() {
- b.resetReaderIndex();
- return this;
- }
-
- @Override
- public ByteBuf markWriterIndex() {
- b.markWriterIndex();
- return this;
- }
-
- @Override
- public ByteBuf resetWriterIndex() {
- b.resetWriterIndex();
- return this;
- }
-
- @Override
- public ByteBuf discardReadBytes() {
- b.discardReadBytes();
- return this;
- }
-
- @Override
- public ByteBuf discardSomeReadBytes() {
- b.discardSomeReadBytes();
- return this;
- }
-
- @Override
- public ByteBuf ensureWritable(int minWritableBytes) {
- b.ensureWritable(minWritableBytes);
- return this;
- }
-
- @Override
- public int ensureWritable(int minWritableBytes, boolean force) {
- return b.ensureWritable(minWritableBytes, false);
- }
-
- @Override
- public boolean getBoolean(int index) {
- return b.getBoolean(index);
- }
-
- @Override
- public byte getByte(int index) {
- return b.getByte(index);
- }
-
- @Override
- public short getUnsignedByte(int index) {
- return b.getUnsignedByte(index);
- }
-
- @Override
- public short getShort(int index) {
- return b.getShort(index);
- }
-
- @Override
- public int getUnsignedShort(int index) {
- return b.getUnsignedShort(index);
- }
-
- @Override
- public int getMedium(int index) {
- return b.getMedium(index);
- }
-
- @Override
- public int getUnsignedMedium(int index) {
- return b.getUnsignedMedium(index);
- }
-
- @Override
- public int getInt(int index) {
- return b.getInt(index);
- }
-
- @Override
- public long getUnsignedInt(int index) {
- return b.getUnsignedInt(index);
- }
-
- @Override
- public long getLong(int index) {
- return b.getLong(index);
- }
-
- @Override
- public char getChar(int index) {
- return b.getChar(index);
- }
-
- @Override
- public float getFloat(int index) {
- return b.getFloat(index);
- }
-
- @Override
- public double getDouble(int index) {
- return b.getDouble(index);
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst) {
- b.getBytes(index, dst);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst, int length) {
- b.getBytes(index, dst, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
- b.getBytes(index, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, byte[] dst) {
- b.getBytes(index, dst);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
- b.getBytes(index, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuffer dst) {
- b.getBytes(index, dst);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
- b.getBytes(index, out, length);
- return this;
- }
-
- @Override
- public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
- return b.getBytes(index, out, length);
- }
-
- @Override
- public ByteBuf setBoolean(int index, boolean value) {
- b.setBoolean(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setByte(int index, int value) {
- b.setByte(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setShort(int index, int value) {
- b.setShort(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setMedium(int index, int value) {
- b.setMedium(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setInt(int index, int value) {
- b.setInt(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setLong(int index, long value) {
- b.setLong(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setChar(int index, int value) {
- b.setChar(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setFloat(int index, float value) {
- b.setFloat(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setDouble(int index, double value) {
- b.setDouble(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src) {
- b.setBytes(index, src);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src, int length) {
- b.setBytes(index, src, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
- b.setBytes(index, src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, byte[] src) {
- b.setBytes(index, src);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
- b.setBytes(index, src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuffer src) {
- b.setBytes(index, src);
- return this;
- }
-
- @Override
- public int setBytes(int index, InputStream in, int length) throws IOException {
- return b.setBytes(index, in, length);
- }
-
- @Override
- public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
- return b.setBytes(index, in, length);
- }
-
- @Override
- public ByteBuf setZero(int index, int length) {
- b.setZero(index, length);
- return this;
- }
-
- @Override
- public boolean readBoolean() {
- return b.readBoolean();
- }
-
- @Override
- public byte readByte() {
- return b.readByte();
- }
-
- @Override
- public short readUnsignedByte() {
- return b.readUnsignedByte();
- }
-
- @Override
- public short readShort() {
- return b.readShort();
- }
-
- @Override
- public int readUnsignedShort() {
- return b.readUnsignedShort();
- }
-
- @Override
- public int readMedium() {
- return b.readMedium();
- }
-
- @Override
- public int readUnsignedMedium() {
- return b.readUnsignedMedium();
- }
-
- @Override
- public int readInt() {
- return b.readInt();
- }
-
- @Override
- public long readUnsignedInt() {
- return b.readUnsignedInt();
- }
-
- @Override
- public long readLong() {
- return b.readLong();
- }
-
- @Override
- public char readChar() {
- return b.readChar();
- }
-
- @Override
- public float readFloat() {
- return b.readFloat();
- }
-
- @Override
- public double readDouble() {
- return b.readDouble();
- }
-
- @Override
- public ByteBuf readBytes(int length) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf readSlice(int length) {
- ByteBuf slice = slice(readerIndex(), length);
- readerIndex(readerIndex() + length);
- return slice;
- }
-
-
- @Override
- public ByteBuf readBytes(ByteBuf dst) {
- b.readBytes(dst);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(ByteBuf dst, int length) {
- b.readBytes(dst, length);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(ByteBuf dst, int dstIndex, int length) {
- b.readBytes(dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(byte[] dst) {
- b.readBytes(dst);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(byte[] dst, int dstIndex, int length) {
- b.readBytes(dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(ByteBuffer dst) {
- b.readBytes(dst);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(OutputStream out, int length) throws IOException {
- b.readBytes(out, length);
- return null;
- }
-
- @Override
- public int readBytes(GatheringByteChannel out, int length) throws IOException {
- return b.readBytes(out, length);
- }
-
- @Override
- public ByteBuf skipBytes(int length) {
- b.skipBytes(length);
- return this;
- }
-
- @Override
- public ByteBuf writeBoolean(boolean value) {
- b.writeBoolean(value);
- return this;
- }
-
- @Override
- public ByteBuf writeByte(int value) {
- b.writeByte(value);
- return this;
- }
-
- @Override
- public ByteBuf writeShort(int value) {
- b.writeShort(value);
- return this;
- }
-
- @Override
- public ByteBuf writeMedium(int value) {
- b.writeMedium(value);
- return this;
- }
-
- @Override
- public ByteBuf writeInt(int value) {
- b.writeInt(value);
- return this;
- }
-
- @Override
- public ByteBuf writeLong(long value) {
- b.writeLong(value);
- return this;
- }
-
- @Override
- public ByteBuf writeChar(int value) {
- b.writeChar(value);
- return this;
- }
-
- @Override
- public ByteBuf writeFloat(float value) {
- b.writeFloat(value);
- return this;
- }
-
- @Override
- public ByteBuf writeDouble(double value) {
- b.writeDouble(value);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(ByteBuf src) {
- b.writeBytes(src);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(ByteBuf src, int length) {
- b.writeBytes(src, length);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(ByteBuf src, int srcIndex, int length) {
- b.writeBytes(src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(byte[] src) {
- b.writeBytes(src);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(byte[] src, int srcIndex, int length) {
- b.writeBytes(src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(ByteBuffer src) {
- b.writeBytes(src);
- return this;
- }
-
- @Override
- public int writeBytes(InputStream in, int length) throws IOException {
- return b.writeBytes(in, length);
- }
-
- @Override
- public int writeBytes(ScatteringByteChannel in, int length) throws IOException {
- return b.writeBytes(in, length);
- }
-
- @Override
- public ByteBuf writeZero(int length) {
- b.writeZero(length);
- return this;
- }
-
- @Override
- public int indexOf(int fromIndex, int toIndex, byte value) {
- return b.indexOf(fromIndex, toIndex, value);
- }
-
- @Override
- public int bytesBefore(byte value) {
- return b.bytesBefore(value);
- }
-
- @Override
- public int bytesBefore(int length, byte value) {
- return b.bytesBefore(length, value);
- }
-
- @Override
- public int bytesBefore(int index, int length, byte value) {
- return b.bytesBefore(index, length, value);
- }
-
- @Override
- public int forEachByte(ByteBufProcessor processor) {
- return b.forEachByte(processor);
- }
-
- @Override
- public int forEachByte(int index, int length, ByteBufProcessor processor) {
- return b.forEachByte(index, length, processor);
- }
-
- @Override
- public int forEachByteDesc(ByteBufProcessor processor) {
- return b.forEachByteDesc(processor);
- }
-
- @Override
- public int forEachByteDesc(int index, int length, ByteBufProcessor processor) {
- return b.forEachByteDesc(index, length, processor);
- }
-
- @Override
- public ByteBuf copy() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf copy(int index, int length) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf slice() {
- return slice(b.readerIndex(), readableBytes());
- }
-
- @Override
- public ByteBuf slice(int index, int length) {
- if (length == 0) {
- return Unpooled.EMPTY_BUFFER;
- }
-
- return new SlicedByteBuf(this, index, length);
- }
-
- @Override
- public ByteBuf duplicate() {
- return new DuplicatedByteBuf(this);
- }
-
- @Override
- public int nioBufferCount() {
- return b.nioBufferCount();
- }
-
- @Override
- public ByteBuffer nioBuffer() {
- return b.nioBuffer();
- }
-
- @Override
- public ByteBuffer nioBuffer(int index, int length) {
- return b.nioBuffer(index, length);
- }
-
- @Override
- public ByteBuffer internalNioBuffer(int index, int length) {
- return b.internalNioBuffer(index, length);
- }
-
- @Override
- public ByteBuffer[] nioBuffers() {
- return b.nioBuffers();
- }
-
- @Override
- public ByteBuffer[] nioBuffers(int index, int length) {
- return b.nioBuffers(index, length);
- }
-
- @Override
- public boolean hasArray() {
- return b.hasArray();
- }
-
- @Override
- public byte[] array() {
- return b.array();
- }
-
- @Override
- public int arrayOffset() {
- return b.arrayOffset();
- }
-
- @Override
- public boolean hasMemoryAddress() {
- return b.hasMemoryAddress();
- }
-
- @Override
- public long memoryAddress() {
- return b.memoryAddress();
- }
-
- @Override
- public String toString(Charset charset) {
- return b.toString(charset);
- }
-
- @Override
- public String toString(int index, int length, Charset charset) {
- return b.toString(index, length, charset);
- }
-
- @Override
- public int hashCode() {
- return System.identityHashCode(this);
- }
-
- @Override
- public boolean equals(Object obj) {
- // identity equals only.
- return this == obj;
- }
-
- @Override
- public String toString() {
- return "AccountingByteBuf [Inner buffer=" + b + ", size=" + size + "]";
- }
-
- @Override
- public int compareTo(ByteBuf buffer) {
- return b.compareTo(buffer);
- }
-
-
- @Override
- public ByteBuf retain(int increment) {
- b.retain(increment);
- return this;
- }
-
- @Override
- public ByteBuf retain() {
- b.retain();
- return this;
- }
-
-
-}
[03/14] git commit: DRILL-1069: Rename RandomReceiver to
UnorderedRecevier.
Posted by ja...@apache.org.
DRILL-1069: Rename RandomReceiver to UnorderedRecevier.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/16808f4a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/16808f4a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/16808f4a
Branch: refs/heads/master
Commit: 16808f4a485418d2974b0be17c2aa9fab2514d38
Parents: 8773f84
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Jun 21 11:24:42 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:08:41 2014 -0700
----------------------------------------------------------------------
.../physical/base/AbstractPhysicalVisitor.java | 4 +-
.../exec/physical/base/PhysicalVisitor.java | 4 +-
.../exec/physical/config/BroadcastExchange.java | 2 +-
.../physical/config/HashToRandomExchange.java | 2 +-
.../config/OrderedPartitionExchange.java | 2 +-
.../exec/physical/config/RandomReceiver.java | 64 -------
.../exec/physical/config/UnionExchange.java | 2 +-
.../exec/physical/config/UnorderedReceiver.java | 64 +++++++
.../physical/impl/RandomReceiverCreator.java | 46 -----
.../exec/physical/impl/WireRecordBatch.java | 181 -------------------
.../UnorderedReceiverBatch.java | 181 +++++++++++++++++++
.../UnorderedReceiverCreator.java | 47 +++++
protocol/src/main/protobuf/UserBitShared.proto | 2 +-
13 files changed, 301 insertions(+), 300 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
index 14cfe81..d7d7c12 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
@@ -29,7 +29,7 @@ import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.physical.config.MergingReceiverPOP;
import org.apache.drill.exec.physical.config.OrderedPartitionSender;
import org.apache.drill.exec.physical.config.Project;
-import org.apache.drill.exec.physical.config.RandomReceiver;
+import org.apache.drill.exec.physical.config.UnorderedReceiver;
import org.apache.drill.exec.physical.config.RangeSender;
import org.apache.drill.exec.physical.config.Screen;
import org.apache.drill.exec.physical.config.SingleSender;
@@ -146,7 +146,7 @@ public abstract class AbstractPhysicalVisitor<T, X, E extends Throwable> impleme
}
@Override
- public T visitRandomReceiver(RandomReceiver op, X value) throws E {
+ public T visitUnorderedReceiver(UnorderedReceiver op, X value) throws E {
return visitReceiver(op, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
index 3eaf026..6d5a6cb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
@@ -29,7 +29,7 @@ import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.physical.config.MergingReceiverPOP;
import org.apache.drill.exec.physical.config.OrderedPartitionSender;
import org.apache.drill.exec.physical.config.Project;
-import org.apache.drill.exec.physical.config.RandomReceiver;
+import org.apache.drill.exec.physical.config.UnorderedReceiver;
import org.apache.drill.exec.physical.config.RangeSender;
import org.apache.drill.exec.physical.config.Screen;
import org.apache.drill.exec.physical.config.SingleSender;
@@ -71,7 +71,7 @@ public interface PhysicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
public RETURN visitHashPartitionSender(HashPartitionSender op, EXTRA value) throws EXCEP;
public RETURN visitOrderedPartitionSender(OrderedPartitionSender op, EXTRA value) throws EXCEP;
- public RETURN visitRandomReceiver(RandomReceiver op, EXTRA value) throws EXCEP;
+ public RETURN visitUnorderedReceiver(UnorderedReceiver op, EXTRA value) throws EXCEP;
public RETURN visitMergingReceiver(MergingReceiverPOP op, EXTRA value) throws EXCEP;
public RETURN visitHashPartitionSender(HashToRandomExchange op, EXTRA value) throws EXCEP;
public RETURN visitRangeSender(RangeSender op, EXTRA value) throws EXCEP;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastExchange.java
index 5e67bcc..2eed4c4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastExchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastExchange.java
@@ -64,7 +64,7 @@ public class BroadcastExchange extends AbstractExchange {
@Override
public Receiver getReceiver(int minorFragmentId) {
- return new RandomReceiver(senderMajorFragmentId, senderLocations);
+ return new UnorderedReceiver(senderMajorFragmentId, senderLocations);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToRandomExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToRandomExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToRandomExchange.java
index 7ad2b65..dddaf83 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToRandomExchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToRandomExchange.java
@@ -71,7 +71,7 @@ public class HashToRandomExchange extends AbstractExchange{
@Override
public Receiver getReceiver(int minorFragmentId) {
- return new RandomReceiver(senderMajorFragmentId, senderLocations);
+ return new UnorderedReceiver(senderMajorFragmentId, senderLocations);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionExchange.java
index 2b80262..8e1526a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionExchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionExchange.java
@@ -93,7 +93,7 @@ public class OrderedPartitionExchange extends AbstractExchange {
@Override
public Receiver getReceiver(int minorFragmentId) {
- return new RandomReceiver(senderMajorFragmentId, senderLocations);
+ return new UnorderedReceiver(senderMajorFragmentId, senderLocations);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/RandomReceiver.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/RandomReceiver.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/RandomReceiver.java
deleted file mode 100644
index b04db40..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/RandomReceiver.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.config;
-
-import java.util.List;
-
-import org.apache.drill.exec.physical.base.AbstractReceiver;
-import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-@JsonTypeName("random-receiver")
-public class RandomReceiver extends AbstractReceiver{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RandomReceiver.class);
-
- private List<DrillbitEndpoint> senders;
-
- @JsonCreator
- public RandomReceiver(@JsonProperty("sender-major-fragment") int oppositeMajorFragmentId,
- @JsonProperty("senders") List<DrillbitEndpoint> senders) {
- super(oppositeMajorFragmentId);
- this.senders = senders;
- }
-
- @Override
- @JsonProperty("senders")
- public List<DrillbitEndpoint> getProvidingEndpoints() {
- return senders;
- }
-
- @Override
- public boolean supportsOutOfOrderExchange() {
- return true;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
- return physicalVisitor.visitRandomReceiver(this, value);
- }
-
- @Override
- public int getOperatorType() {
- return CoreOperatorType.RANDOM_RECEIVER_VALUE;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionExchange.java
index a638741..cafdbdd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionExchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionExchange.java
@@ -59,7 +59,7 @@ public class UnionExchange extends AbstractExchange{
@Override
public Receiver getReceiver(int minorFragmentId) {
- return new RandomReceiver(this.senderMajorFragmentId, senderLocations);
+ return new UnorderedReceiver(this.senderMajorFragmentId, senderLocations);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnorderedReceiver.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnorderedReceiver.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnorderedReceiver.java
new file mode 100644
index 0000000..a204752
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnorderedReceiver.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.config;
+
+import java.util.List;
+
+import org.apache.drill.exec.physical.base.AbstractReceiver;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("unordered-receiver")
+public class UnorderedReceiver extends AbstractReceiver{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UnorderedReceiver.class);
+
+ private List<DrillbitEndpoint> senders;
+
+ @JsonCreator
+ public UnorderedReceiver(@JsonProperty("sender-major-fragment") int oppositeMajorFragmentId,
+ @JsonProperty("senders") List<DrillbitEndpoint> senders) {
+ super(oppositeMajorFragmentId);
+ this.senders = senders;
+ }
+
+ @Override
+ @JsonProperty("senders")
+ public List<DrillbitEndpoint> getProvidingEndpoints() {
+ return senders;
+ }
+
+ @Override
+ public boolean supportsOutOfOrderExchange() {
+ return true;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
+ return physicalVisitor.visitUnorderedReceiver(this, value);
+ }
+
+ @Override
+ public int getOperatorType() {
+ return CoreOperatorType.RANDOM_RECEIVER_VALUE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RandomReceiverCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RandomReceiverCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RandomReceiverCreator.java
deleted file mode 100644
index 4ff5831..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RandomReceiverCreator.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl;
-
-import java.util.List;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.config.RandomReceiver;
-import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.work.batch.IncomingBuffers;
-import org.apache.drill.exec.work.batch.RawBatchBuffer;
-
-public class RandomReceiverCreator implements BatchCreator<RandomReceiver>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RandomReceiverCreator.class);
-
- @Override
- public RecordBatch getBatch(FragmentContext context, RandomReceiver receiver, List<RecordBatch> children)
- throws ExecutionSetupException {
- assert children == null || children.isEmpty();
- IncomingBuffers bufHolder = context.getBuffers();
- assert bufHolder != null : "IncomingBuffers must be defined for any place a receiver is declared.";
-
- RawBatchBuffer[] buffers = bufHolder.getBuffers(receiver.getOppositeMajorFragmentId());
- assert buffers.length == 1;
- RawBatchBuffer buffer = buffers[0];
- return new WireRecordBatch(context, buffer, receiver);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
deleted file mode 100644
index 1eae0c9..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.ops.OpProfileDef;
-import org.apache.drill.exec.ops.OperatorStats;
-import org.apache.drill.exec.physical.config.RandomReceiver;
-import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
-import org.apache.drill.exec.record.BatchSchema;
-import org.apache.drill.exec.record.RawFragmentBatch;
-import org.apache.drill.exec.record.RawFragmentBatchProvider;
-import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.TypedFieldId;
-import org.apache.drill.exec.record.VectorContainer;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.record.WritableBatch;
-import org.apache.drill.exec.record.selection.SelectionVector2;
-import org.apache.drill.exec.record.selection.SelectionVector4;
-
-public class WireRecordBatch implements RecordBatch {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WireRecordBatch.class);
-
- private RecordBatchLoader batchLoader;
- private RawFragmentBatchProvider fragProvider;
- private FragmentContext context;
- private BatchSchema schema;
- private OperatorStats stats;
- private boolean first = true;
-
-
- public WireRecordBatch(FragmentContext context, RawFragmentBatchProvider fragProvider, RandomReceiver config) throws OutOfMemoryException {
- this.fragProvider = fragProvider;
- this.context = context;
- // In normal case, batchLoader does not require an allocator. However, in case of splitAndTransfer of a value vector,
- // we may need an allocator for the new offset vector. Therefore, here we pass the context's allocator to batchLoader.
- this.batchLoader = new RecordBatchLoader(context.getAllocator());
- this.stats = context.getStats().getOperatorStats(new OpProfileDef(config.getOperatorId(), config.getOperatorType(), 0));
- }
-
- @Override
- public FragmentContext getContext() {
- return context;
- }
-
- @Override
- public BatchSchema getSchema() {
- return schema;
- }
-
- @Override
- public int getRecordCount() {
- return batchLoader.getRecordCount();
- }
-
- @Override
- public void kill() {
- fragProvider.kill(context);
- }
-
- @Override
- public Iterator<VectorWrapper<?>> iterator() {
- return batchLoader.iterator();
- }
-
- @Override
- public SelectionVector2 getSelectionVector2() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SelectionVector4 getSelectionVector4() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public TypedFieldId getValueVectorId(SchemaPath path) {
- return batchLoader.getValueVectorId(path);
- }
-
- @Override
- public VectorWrapper<?> getValueAccessorById(Class<?> clazz, int... ids) {
- return batchLoader.getValueAccessorById(clazz, ids);
- }
-
- @Override
- public IterOutcome next() {
- stats.startProcessing();
- try{
- RawFragmentBatch batch;
- try {
- stats.startWait();
- batch = fragProvider.getNext();
-
- // skip over empty batches. we do this since these are basically control messages.
- while(batch != null && !batch.getHeader().getIsOutOfMemory() && batch.getHeader().getDef().getRecordCount() == 0 && !first){
- if (first) {
- first = false;
- RecordBatchDef rbd = batch.getHeader().getDef();
- batchLoader.load(rbd, batch.getBody());
- batch.release();
- schema = batchLoader.getSchema().clone();
- batchLoader.clear();
- }
- batch = fragProvider.getNext();
- }
- } finally {
- stats.stopWait();
- }
-
- first = false;
-
- if (batch == null){
- batchLoader.clear();
- return IterOutcome.NONE;
- }
-
- if (batch.getHeader().getIsOutOfMemory()) {
- return IterOutcome.OUT_OF_MEMORY;
- }
-
-
-// logger.debug("Next received batch {}", batch);
-
- RecordBatchDef rbd = batch.getHeader().getDef();
- boolean schemaChanged = batchLoader.load(rbd, batch.getBody());
-// System.out.println(rbd.getRecordCount());
- batch.release();
- if(schemaChanged){
- this.schema = batchLoader.getSchema();
- return IterOutcome.OK_NEW_SCHEMA;
- }else{
- return IterOutcome.OK;
- }
- }catch(SchemaChangeException | IOException ex){
- context.fail(ex);
- return IterOutcome.STOP;
- } finally {
- stats.stopProcessing();
- }
- }
-
- @Override
- public WritableBatch getWritableBatch() {
- return batchLoader.getWritableBatch();
- }
-
- @Override
- public void cleanup() {
- batchLoader.clear();
- fragProvider.cleanup();
- }
-
- @Override
- public VectorContainer getOutgoingContainer() {
- throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverBatch.java
new file mode 100644
index 0000000..3d9e470
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverBatch.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.unorderedreceiver;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OpProfileDef;
+import org.apache.drill.exec.ops.OperatorStats;
+import org.apache.drill.exec.physical.config.UnorderedReceiver;
+import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.RawFragmentBatch;
+import org.apache.drill.exec.record.RawFragmentBatchProvider;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
+import org.apache.drill.exec.record.selection.SelectionVector2;
+import org.apache.drill.exec.record.selection.SelectionVector4;
+
+public class UnorderedReceiverBatch implements RecordBatch {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UnorderedReceiverBatch.class);
+
+ private RecordBatchLoader batchLoader;
+ private RawFragmentBatchProvider fragProvider;
+ private FragmentContext context;
+ private BatchSchema schema;
+ private OperatorStats stats;
+ private boolean first = true;
+
+
+ public UnorderedReceiverBatch(FragmentContext context, RawFragmentBatchProvider fragProvider, UnorderedReceiver config) throws OutOfMemoryException {
+ this.fragProvider = fragProvider;
+ this.context = context;
+ // In normal case, batchLoader does not require an allocator. However, in case of splitAndTransfer of a value vector,
+ // we may need an allocator for the new offset vector. Therefore, here we pass the context's allocator to batchLoader.
+ this.batchLoader = new RecordBatchLoader(context.getAllocator());
+ this.stats = context.getStats().getOperatorStats(new OpProfileDef(config.getOperatorId(), config.getOperatorType(), 0), null);
+ }
+
+ @Override
+ public FragmentContext getContext() {
+ return context;
+ }
+
+ @Override
+ public BatchSchema getSchema() {
+ return schema;
+ }
+
+ @Override
+ public int getRecordCount() {
+ return batchLoader.getRecordCount();
+ }
+
+ @Override
+ public void kill() {
+ fragProvider.kill(context);
+ }
+
+ @Override
+ public Iterator<VectorWrapper<?>> iterator() {
+ return batchLoader.iterator();
+ }
+
+ @Override
+ public SelectionVector2 getSelectionVector2() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SelectionVector4 getSelectionVector4() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TypedFieldId getValueVectorId(SchemaPath path) {
+ return batchLoader.getValueVectorId(path);
+ }
+
+ @Override
+ public VectorWrapper<?> getValueAccessorById(Class<?> clazz, int... ids) {
+ return batchLoader.getValueAccessorById(clazz, ids);
+ }
+
+ @Override
+ public IterOutcome next() {
+ stats.startProcessing();
+ try{
+ RawFragmentBatch batch;
+ try {
+ stats.startWait();
+ batch = fragProvider.getNext();
+
+ // skip over empty batches. we do this since these are basically control messages.
+ while(batch != null && !batch.getHeader().getIsOutOfMemory() && batch.getHeader().getDef().getRecordCount() == 0 && !first){
+ if (first) {
+ first = false;
+ RecordBatchDef rbd = batch.getHeader().getDef();
+ batchLoader.load(rbd, batch.getBody());
+ batch.release();
+ schema = batchLoader.getSchema().clone();
+ batchLoader.clear();
+ }
+ batch = fragProvider.getNext();
+ }
+ } finally {
+ stats.stopWait();
+ }
+
+ first = false;
+
+ if (batch == null){
+ batchLoader.clear();
+ return IterOutcome.NONE;
+ }
+
+ if (batch.getHeader().getIsOutOfMemory()) {
+ return IterOutcome.OUT_OF_MEMORY;
+ }
+
+
+// logger.debug("Next received batch {}", batch);
+
+ RecordBatchDef rbd = batch.getHeader().getDef();
+ boolean schemaChanged = batchLoader.load(rbd, batch.getBody());
+// System.out.println(rbd.getRecordCount());
+ batch.release();
+ if(schemaChanged){
+ this.schema = batchLoader.getSchema();
+ return IterOutcome.OK_NEW_SCHEMA;
+ }else{
+ return IterOutcome.OK;
+ }
+ }catch(SchemaChangeException | IOException ex){
+ context.fail(ex);
+ return IterOutcome.STOP;
+ } finally {
+ stats.stopProcessing();
+ }
+ }
+
+ @Override
+ public WritableBatch getWritableBatch() {
+ return batchLoader.getWritableBatch();
+ }
+
+ @Override
+ public void cleanup() {
+ batchLoader.clear();
+ fragProvider.cleanup();
+ }
+
+ @Override
+ public VectorContainer getOutgoingContainer() {
+ throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverCreator.java
new file mode 100644
index 0000000..6be8714
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverCreator.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.unorderedreceiver;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.UnorderedReceiver;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.work.batch.IncomingBuffers;
+import org.apache.drill.exec.work.batch.RawBatchBuffer;
+
+public class UnorderedReceiverCreator implements BatchCreator<UnorderedReceiver>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UnorderedReceiverCreator.class);
+
+ @Override
+ public RecordBatch getBatch(FragmentContext context, UnorderedReceiver receiver, List<RecordBatch> children)
+ throws ExecutionSetupException {
+ assert children == null || children.isEmpty();
+ IncomingBuffers bufHolder = context.getBuffers();
+ assert bufHolder != null : "IncomingBuffers must be defined for any place a receiver is declared.";
+
+ RawBatchBuffer[] buffers = bufHolder.getBuffers(receiver.getOppositeMajorFragmentId());
+ assert buffers.length == 1;
+ RawBatchBuffer buffer = buffers[0];
+ return new UnorderedReceiverBatch(context, buffer, receiver);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/16808f4a/protocol/src/main/protobuf/UserBitShared.proto
----------------------------------------------------------------------
diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto
index fc6f1b5..6fd73db 100644
--- a/protocol/src/main/protobuf/UserBitShared.proto
+++ b/protocol/src/main/protobuf/UserBitShared.proto
@@ -178,7 +178,7 @@ enum CoreOperatorType {
MERGING_RECEIVER = 8;
ORDERED_PARTITION_SENDER = 9;
PROJECT = 10;
- RANDOM_RECEIVER = 11;
+ UNORDERED_RECEIVER = 11;
RANGE_SENDER = 12;
SCREEN = 13;
SELECTION_VECTOR_REMOVER = 14;
[14/14] git commit: Remove deprecated settings from java-exec's
drill-module.conf
Posted by ja...@apache.org.
Remove deprecated settings from java-exec's drill-module.conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/33c28f62
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/33c28f62
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/33c28f62
Branch: refs/heads/master
Commit: 33c28f624df83bfc7195ad33877b3be7c963e0b5
Parents: 45ce71c
Author: Jacques Nadeau <ja...@apache.org>
Authored: Wed Jun 25 09:00:16 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 11:39:08 2014 -0700
----------------------------------------------------------------------
exec/java-exec/src/main/resources/drill-module.conf | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33c28f62/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index 1519327..8a69ec1 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -38,7 +38,7 @@ drill.exec: {
count: 7200,
delay: 500
},
- threads: 4
+ threads: 10
}
},
use.ip : false
@@ -89,10 +89,7 @@ drill.exec: {
start: 35000
},
work: {
- max.width.per.endpoint: 5,
- global.max.width: 100,
- affinity.factor: 1.2,
- executor.threads: 4
+ affinity.factor: 1.2
},
sys.store.provider: {
class: "org.apache.drill.exec.store.sys.zk.ZkPStoreProvider",
@@ -117,7 +114,6 @@ drill.exec: {
size: 100000000
}
},
- cache.hazel.subnets: ["*.*.*.*"],
sort: {
purge.threshold : 100,
external: {
[13/14] git commit: Fix some EPOLL bugs and disable for now.
Posted by ja...@apache.org.
Fix some EPOLL bugs and disable for now.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/45ce71ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/45ce71ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/45ce71ca
Branch: refs/heads/master
Commit: 45ce71ca0a530de55f35d9086f5326a135b22448
Parents: 0b296c8
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Jun 24 15:06:42 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:10:13 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/exec/rpc/BasicClient.java | 6 +++---
.../java/org/apache/drill/exec/rpc/BasicServer.java | 6 +++---
.../org/apache/drill/exec/rpc/TransportCheck.java | 2 +-
.../org/apache/drill/exec/service/ServiceEngine.java | 4 ++--
.../physical/impl/xsort/TestSimpleExternalSort.java | 14 +++++++-------
5 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/45ce71ca/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
index 0b7fd0f..3dc90b9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
@@ -87,9 +87,9 @@ public abstract class BasicClient<T extends EnumLite, R extends RemoteConnection
}
}); //
- if(TransportCheck.SUPPORTS_EPOLL){
- b.option(EpollChannelOption.SO_REUSEPORT, true); //
- }
+// if(TransportCheck.SUPPORTS_EPOLL){
+// b.option(EpollChannelOption.SO_REUSEPORT, true); //
+// }
}
public abstract ProtobufLengthDecoder getDecoder(BufferAllocator allocator);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/45ce71ca/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
index 82d37db..b08685b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
@@ -87,9 +87,9 @@ public abstract class BasicServer<T extends EnumLite, C extends RemoteConnection
}
});
- if(TransportCheck.SUPPORTS_EPOLL){
- b.option(EpollChannelOption.SO_REUSEPORT, true); //
- }
+// if(TransportCheck.SUPPORTS_EPOLL){
+// b.option(EpollChannelOption.SO_REUSEPORT, true); //
+// }
}
public OutOfMemoryHandler getOutOfMemoryHandler() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/45ce71ca/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
index b326a1f..6401518 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
@@ -44,7 +44,7 @@ public class TransportCheck {
String name = SystemPropertyUtil.get("os.name").toLowerCase(Locale.US).trim();
if (!name.startsWith("linux") || SystemPropertyUtil.getBoolean("drill.exec.disable-linux-epoll", false) //
-// /* disable epoll */ || true //
+ /* disable epoll */ || true //
) {
SUPPORTS_EPOLL = false;
}else{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/45ce71ca/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java
index bd745d1..ce109d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java
@@ -29,6 +29,7 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.DrillbitStartupException;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.rpc.NamedThreadFactory;
+import org.apache.drill.exec.rpc.TransportCheck;
import org.apache.drill.exec.rpc.control.Controller;
import org.apache.drill.exec.rpc.control.ControllerImpl;
import org.apache.drill.exec.rpc.control.WorkEventBus;
@@ -52,8 +53,7 @@ public class ServiceEngine implements Closeable{
private final boolean allowPortHunting;
public ServiceEngine(ControlMessageHandler controlMessageHandler, UserWorker userWorker, BootStrapContext context, WorkEventBus workBus, DataResponseHandler dataHandler, boolean allowPortHunting){
- this.userServer = new UserServer(context.getAllocator(), new NioEventLoopGroup(context.getConfig().getInt(ExecConstants.USER_SERVER_RPC_THREADS),
- new NamedThreadFactory("UserServer-")), userWorker);
+ this.userServer = new UserServer(context.getAllocator(), TransportCheck.createEventLoopGroup(context.getConfig().getInt(ExecConstants.USER_SERVER_RPC_THREADS), "UserServer-"), userWorker);
this.controller = new ControllerImpl(context, controlMessageHandler, allowPortHunting);
this.dataPool = new DataConnectionCreator(context, workBus, dataHandler, allowPortHunting);
this.config = context.getConfig();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/45ce71ca/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
index 47f0342..16b1f40 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
@@ -17,8 +17,10 @@
*/
package org.apache.drill.exec.physical.impl.xsort;
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
import org.apache.drill.BaseTestQuery;
import org.apache.drill.common.config.DrillConfig;
@@ -37,12 +39,10 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+@Ignore
public class TestSimpleExternalSort extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleExternalSort.class);
DrillConfig c = DrillConfig.create();
[07/14] DRILL-1067: RPC layer improvements - Unfork Netty and upgrade
to 4.0.20.Final - Create support for EPOLL native impl on Linux (disable by
providing -Ddrill.exec.disable_linux_epoll=false) - Move BitData connection
to Multiplex with DataTunnel lev
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
index 257f6fc..cd08367 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.memory;
+import io.netty.buffer.AccountingByteBuf;
import io.netty.buffer.ByteBuf;
import java.util.Arrays;
@@ -57,6 +58,17 @@ public class Accountor {
}
}
+ public boolean transferTo(Accountor target, AccountingByteBuf buf, long size){
+ boolean withinLimit = target.forceAdditionalReservation(size);
+ release(buf, size);
+
+ if (ENABLE_ACCOUNTING) {
+ target.buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread().getStackTrace()));
+ }
+
+ return withinLimit;
+ }
+
public long getAvailable() {
if (parent != null) {
return Math.min(parent.getAvailable(), getCapacity() - getAllocation());
@@ -76,8 +88,12 @@ public class Accountor {
return remainder.get(size);
}
- public void forceAdditionalReservation(long size) {
- if(size > 0) remainder.forceGet(size);
+ public boolean forceAdditionalReservation(long size) {
+ if(size > 0){
+ return remainder.forceGet(size);
+ }else{
+ return true;
+ }
}
public void reserved(long expected, AccountingByteBuf buf){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
index 1ae1e4c..0d54974 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
@@ -66,13 +66,14 @@ public class AtomicRemainder {
*
* @param size
*/
- public void forceGet(long size) {
- long newAvailableShared = availableShared.addAndGet(size);
-// if (DEBUG)
-// logger.info("Force get {}. a.s. {} a.p. {} hashcode: {}", size, availableShared, availablePrivate, hashCode(), new Exception());
-// assert newAvailableShared <= initShared;
- if (parent != null)
- parent.forceGet(size);
+ public boolean forceGet(long size) {
+ if(get(size)){
+ return true;
+ }else{
+ availableShared.addAndGet(size);
+ if (parent != null) parent.forceGet(size);
+ return false;
+ }
}
public boolean get(long size) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
index 50f93fc..38fe4f1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.memory;
+import io.netty.buffer.AccountingByteBuf;
import io.netty.buffer.ByteBufAllocator;
import java.io.Closeable;
@@ -32,7 +33,7 @@ public interface BufferAllocator extends Closeable {
/**
* Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the
* requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
- *
+ *
* @param size
* The size in bytes.
* @return A new ByteBuf.
@@ -42,7 +43,7 @@ public interface BufferAllocator extends Closeable {
/**
* Allocate a new or reused buffer within provided range. Note that the buffer may technically be larger than the
* requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
- *
+ *
* @param minSize The minimum size in bytes.
* @param maxSize The maximum size in bytes.
* @return A new ByteBuf.
@@ -54,6 +55,14 @@ public interface BufferAllocator extends Closeable {
public abstract BufferAllocator getChildAllocator(FragmentHandle handle, long initialReservation,
long maximumReservation) throws OutOfMemoryException;
+ /**
+ * Take over ownership of fragment accounting. Always takes over ownership.
+ * @param buf
+ * @return false if over allocation.
+ */
+ public boolean takeOwnership(AccountingByteBuf buf) ;
+
+
public PreAllocator getNewPreAllocator();
/**
@@ -71,7 +80,7 @@ public interface BufferAllocator extends Closeable {
*/
/**
- *
+ *
*/
/**
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
index ecbf0b4..e785f0d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
@@ -17,10 +17,11 @@
*/
package org.apache.drill.exec.memory;
+import io.netty.buffer.AccountingByteBuf;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.PooledByteBufAllocatorL;
-import io.netty.buffer.PooledUnsafeDirectByteBufL;
import java.util.HashMap;
import java.util.HashSet;
@@ -37,7 +38,7 @@ public class TopLevelAllocator implements BufferAllocator {
private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled();
private final Set<ChildAllocator> children;
- private final PooledByteBufAllocatorL innerAllocator = PooledByteBufAllocatorL.DEFAULT;
+ private final PooledByteBufAllocator innerAllocator = PooledByteBufAllocatorL.DEFAULT;
private final Accountor acct;
private final boolean errorOnLeak;
@@ -63,10 +64,15 @@ public class TopLevelAllocator implements BufferAllocator {
);
}
+ @Override
+ public boolean takeOwnership(AccountingByteBuf buf) {
+ return buf.transferAccounting(acct);
+ }
+
public AccountingByteBuf buffer(int min, int max) {
if(!acct.reserve(min)) return null;
ByteBuf buffer = innerAllocator.directBuffer(min, max);
- AccountingByteBuf wrapped = new AccountingByteBuf(acct, (PooledUnsafeDirectByteBufL) buffer);
+ AccountingByteBuf wrapped = new AccountingByteBuf(acct, buffer);
acct.reserved(min, wrapped);
return wrapped;
}
@@ -124,6 +130,11 @@ public class TopLevelAllocator implements BufferAllocator {
}
@Override
+ public boolean takeOwnership(AccountingByteBuf buf) {
+ return buf.transferAccounting(childAcct);
+ }
+
+ @Override
public AccountingByteBuf buffer(int size, int max) {
if(!childAcct.reserve(size)){
logger.warn("Unable to allocate buffer of size {} due to memory limit. Current allocation: {}", size, getAllocatedMemory());
@@ -131,7 +142,7 @@ public class TopLevelAllocator implements BufferAllocator {
};
ByteBuf buffer = innerAllocator.directBuffer(size, max);
- AccountingByteBuf wrapped = new AccountingByteBuf(childAcct, (PooledUnsafeDirectByteBufL) buffer);
+ AccountingByteBuf wrapped = new AccountingByteBuf(childAcct, buffer);
childAcct.reserved(buffer.capacity(), wrapped);
return wrapped;
}
@@ -227,7 +238,7 @@ public class TopLevelAllocator implements BufferAllocator {
public AccountingByteBuf getAllocation(){
- AccountingByteBuf b = new AccountingByteBuf(acct, (PooledUnsafeDirectByteBufL) innerAllocator.buffer(bytes));
+ AccountingByteBuf b = new AccountingByteBuf(acct, innerAllocator.buffer(bytes));
acct.reserved(bytes, b);
return b;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/proto/helper/QueryIdHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/proto/helper/QueryIdHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/proto/helper/QueryIdHelper.java
index 749b3d5..3bbe231 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/proto/helper/QueryIdHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/proto/helper/QueryIdHelper.java
@@ -19,14 +19,19 @@
package org.apache.drill.exec.proto.helper;
import java.util.UUID;
+
+import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
/* Helper class around the QueryId protobuf */
public class QueryIdHelper {
- /* Generate a UUID from the two parts of the queryid */
- public static String getQueryId(QueryId queryId)
- {
- return (new UUID(queryId.getPart1(), queryId.getPart2())).toString();
- }
+ /* Generate a UUID from the two parts of the queryid */
+ public static String getQueryId(QueryId queryId) {
+ return (new UUID(queryId.getPart1(), queryId.getPart2())).toString();
+ }
+
+ public static String getQueryIdentifier(FragmentHandle h) {
+ return getQueryId(h.getQueryId()) + ":" + h.getMajorFragmentId() + ":" + h.getMinorFragmentId();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java
index a7d3f4b..39eae07 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java
@@ -21,6 +21,8 @@ import io.netty.buffer.ByteBuf;
import org.apache.drill.exec.proto.BitData.FragmentRecordBatch;
import org.apache.drill.exec.rpc.RemoteConnection;
+import org.apache.drill.exec.rpc.ResponseSender;
+import org.apache.drill.exec.rpc.data.DataRpcConfig;
public class RawFragmentBatch {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RawFragmentBatch.class);
@@ -28,12 +30,14 @@ public class RawFragmentBatch {
final RemoteConnection connection;
final FragmentRecordBatch header;
final ByteBuf body;
+ final ResponseSender sender;
- public RawFragmentBatch(RemoteConnection connection, FragmentRecordBatch header, ByteBuf body) {
+ public RawFragmentBatch(RemoteConnection connection, FragmentRecordBatch header, ByteBuf body, ResponseSender sender) {
super();
this.header = header;
this.body = body;
this.connection = connection;
+ this.sender = sender;
if(body != null) body.retain();
}
@@ -49,7 +53,7 @@ public class RawFragmentBatch {
public String toString() {
return "RawFragmentBatch [header=" + header + ", body=" + body + "]";
}
-
+
public void release(){
if(body != null) body.release();
}
@@ -58,5 +62,12 @@ public class RawFragmentBatch {
return connection;
}
-
+ public ResponseSender getSender() {
+ return sender;
+ }
+
+ public void sendOk(){
+ sender.send(DataRpcConfig.OK);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
index 562fa90..0b7fd0f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
@@ -28,6 +28,7 @@ import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
+import io.netty.channel.epoll.EpollChannelOption;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.util.concurrent.GenericFutureListener;
@@ -59,8 +60,10 @@ public abstract class BasicClient<T extends EnumLite, R extends RemoteConnection
b = new Bootstrap() //
.group(eventLoopGroup) //
- .channel(NioSocketChannel.class) //
+ .channel(TransportCheck.getClientSocketChannel()) //
.option(ChannelOption.ALLOCATOR, alloc) //
+ .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 30*1000)
+ .option(ChannelOption.SO_REUSEADDR, true)
.option(ChannelOption.SO_RCVBUF, 1 << 17) //
.option(ChannelOption.SO_SNDBUF, 1 << 17) //
.option(ChannelOption.TCP_NODELAY, true)
@@ -82,9 +85,11 @@ public abstract class BasicClient<T extends EnumLite, R extends RemoteConnection
);
connect = true;
}
- }) //
+ }); //
- ;
+ if(TransportCheck.SUPPORTS_EPOLL){
+ b.option(EpollChannelOption.SO_REUSEPORT, true); //
+ }
}
public abstract ProtobufLengthDecoder getDecoder(BufferAllocator allocator);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
index 599f0fd..82d37db 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
@@ -25,10 +25,11 @@ import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
+import io.netty.channel.epoll.EpollChannelOption;
+import io.netty.channel.epoll.EpollServerSocketChannel;
+import io.netty.channel.socket.ServerSocketChannel;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.logging.LogLevel;
-import io.netty.handler.logging.LoggingHandler;
import java.io.IOException;
import java.net.BindException;
@@ -56,11 +57,12 @@ public abstract class BasicServer<T extends EnumLite, C extends RemoteConnection
public BasicServer(RpcConfig rpcMapping, ByteBufAllocator alloc, EventLoopGroup eventLoopGroup) {
super(rpcMapping);
this.eventLoopGroup = eventLoopGroup;
-
b = new ServerBootstrap() //
- .channel(NioServerSocketChannel.class) //
- .option(ChannelOption.SO_BACKLOG, 100) //
+ .channel(TransportCheck.getServerSocketChannel()) //
+ .option(ChannelOption.SO_BACKLOG, 1000) //
+ .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 30*1000)
.option(ChannelOption.TCP_NODELAY, true)
+ .option(ChannelOption.SO_REUSEADDR, true)
.option(ChannelOption.SO_RCVBUF, 1 << 17) //
.option(ChannelOption.SO_SNDBUF, 1 << 17) //
.group(eventLoopGroup) //
@@ -84,6 +86,10 @@ public abstract class BasicServer<T extends EnumLite, C extends RemoteConnection
// logger.debug("Server connection initialization completed.");
}
});
+
+ if(TransportCheck.SUPPORTS_EPOLL){
+ b.option(EpollChannelOption.SO_REUSEPORT, true); //
+ }
}
public OutOfMemoryHandler getOutOfMemoryHandler() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java
new file mode 100644
index 0000000..6dc9ae1
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+
+public interface ResponseSender {
+ public void send(Response r);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
index 30101b2..05cca5a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.rpc;
import com.google.common.base.Stopwatch;
+
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufInputStream;
import io.netty.channel.Channel;
@@ -45,7 +46,7 @@ import com.google.protobuf.Parser;
/**
* The Rpc Bus deals with incoming and outgoing communication and is used on both the server and the client side of a
* system.
- *
+ *
* @param <T>
*/
public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> implements Closeable {
@@ -55,6 +56,10 @@ public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> imp
protected abstract MessageLite getResponseDefaultInstance(int rpcType) throws RpcException;
+ protected void handle(C connection, int rpcType, ByteBuf pBody, ByteBuf dBody, ResponseSender sender) throws RpcException{
+ sender.send(handle(connection, rpcType, pBody, dBody));
+ }
+
protected abstract Response handle(C connection, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException;
public abstract boolean isClient();
@@ -70,19 +75,19 @@ public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> imp
DrillRpcFutureImpl<RECEIVE> rpcFuture = new DrillRpcFutureImpl<RECEIVE>();
this.send(rpcFuture, connection, rpcType, protobufBody, clazz, dataBodies);
return rpcFuture;
- }
-
+ }
+
public <SEND extends MessageLite, RECEIVE extends MessageLite> void send(RpcOutcomeListener<RECEIVE> listener, C connection, T rpcType,
SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
send(listener, connection, rpcType, protobufBody, clazz, false, dataBodies);
}
-
+
public <SEND extends MessageLite, RECEIVE extends MessageLite> void send(RpcOutcomeListener<RECEIVE> listener, C connection, T rpcType,
SEND protobufBody, Class<RECEIVE> clazz, boolean allowInEventLoop, ByteBuf... dataBodies) {
-
+
if(!allowInEventLoop){
if(connection.inEventLoop()) throw new IllegalStateException("You attempted to send while inside the rpc event thread. This isn't allowed because sending will block if the channel is backed up.");
-
+
if(!connection.blockOnNotWritable(listener)) return;
}
@@ -136,6 +141,27 @@ public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> imp
return new ChannelClosedHandler();
}
+ private class ResponseSenderImpl implements ResponseSender {
+
+ RemoteConnection connection;
+ int coordinationId;
+
+ public ResponseSenderImpl(RemoteConnection connection, int coordinationId) {
+ super();
+ this.connection = connection;
+ this.coordinationId = coordinationId;
+ }
+
+ public void send(Response r){
+ assert rpcConfig.checkResponseSend(r.rpcType, r.pBody.getClass());
+ OutboundRpcMessage outMessage = new OutboundRpcMessage(RpcMode.RESPONSE, r.rpcType, coordinationId,
+ r.pBody, r.dBodies);
+ if (RpcConstants.EXTRA_DEBUGGING) logger.debug("Adding message to outbound buffer. {}", outMessage);
+ connection.getChannel().writeAndFlush(outMessage);
+ }
+
+ }
+
protected class InboundHandler extends MessageToMessageDecoder<InboundRpcMessage> {
private final C connection;
@@ -151,13 +177,9 @@ public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> imp
switch (msg.mode) {
case REQUEST: {
// handle message and ack.
- Response r = handle(connection, msg.rpcType, msg.pBody, msg.dBody);
+ ResponseSender sender = new ResponseSenderImpl(connection, msg.coordinationId);
+ handle(connection, msg.rpcType, msg.pBody, msg.dBody, sender);
msg.release(); // we release our ownership. Handle could have taken over ownership.
- assert rpcConfig.checkResponseSend(r.rpcType, r.pBody.getClass());
- OutboundRpcMessage outMessage = new OutboundRpcMessage(RpcMode.RESPONSE, r.rpcType, msg.coordinationId,
- r.pBody, r.dBodies);
- if (RpcConstants.EXTRA_DEBUGGING) logger.debug("Adding message to outbound buffer. {}", outMessage);
- ctx.writeAndFlush(outMessage);
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
new file mode 100644
index 0000000..b326a1f
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.epoll.EpollEventLoopGroup;
+import io.netty.channel.epoll.EpollServerSocketChannel;
+import io.netty.channel.epoll.EpollSocketChannel;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.ServerSocketChannel;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.util.internal.SystemPropertyUtil;
+
+import java.util.Locale;
+
+/**
+ * TransportCheck decides whether or not to use the native EPOLL mechanism for communication.
+ */
+@SuppressWarnings("unused")
+public class TransportCheck {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TransportCheck.class);
+
+ public static final boolean SUPPORTS_EPOLL;
+
+ static{
+
+ String name = SystemPropertyUtil.get("os.name").toLowerCase(Locale.US).trim();
+
+ if (!name.startsWith("linux") || SystemPropertyUtil.getBoolean("drill.exec.disable-linux-epoll", false) //
+// /* disable epoll */ || true //
+ ) {
+ SUPPORTS_EPOLL = false;
+ }else{
+ SUPPORTS_EPOLL = true;
+ }
+ }
+
+ public static Class<? extends ServerSocketChannel> getServerSocketChannel(){
+ if(SUPPORTS_EPOLL){
+ return EpollServerSocketChannel.class;
+ }else{
+ return NioServerSocketChannel.class;
+ }
+ }
+
+ public static Class<? extends SocketChannel> getClientSocketChannel(){
+ if(SUPPORTS_EPOLL){
+ return EpollSocketChannel.class;
+ }else{
+ return NioSocketChannel.class;
+ }
+ }
+
+ public static EventLoopGroup createEventLoopGroup(int nThreads, String prefix) {
+ if(SUPPORTS_EPOLL){
+ return new EpollEventLoopGroup(nThreads, new NamedThreadFactory(prefix));
+ }else{
+ return new NioEventLoopGroup(nThreads, new NamedThreadFactory(prefix));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/BitServerConnection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/BitServerConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/BitServerConnection.java
index 58fa403..4583f88 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/BitServerConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/BitServerConnection.java
@@ -17,110 +17,24 @@
*/
package org.apache.drill.exec.rpc.data;
-import io.netty.buffer.ByteBufAllocator;
import io.netty.channel.Channel;
-import org.apache.drill.exec.memory.AccountingByteBuf;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.rpc.RemoteConnection;
import org.apache.drill.exec.work.fragment.FragmentManager;
public class BitServerConnection extends RemoteConnection{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitServerConnection.class);
- private AllocatorProxy proxy = new AllocatorProxy();
- private volatile FragmentManager manager;
-
- public BitServerConnection(Channel channel, BufferAllocator initialAllocator) {
+ private final BufferAllocator allocator;
+ public BitServerConnection(Channel channel, BufferAllocator allocator) {
super(channel);
- proxy.setAllocator(initialAllocator);
- }
-
- void setManager(FragmentManager manager){
- this.manager = manager;
- if (manager != null) { // Do this check for TestBitRpc test
- this.proxy.setAllocator(manager.getFragmentContext().getAllocator());
- manager.addConnection(this);
- }
+ this.allocator = allocator;
}
@Override
public BufferAllocator getAllocator() {
- if(manager != null){
- return manager.getFragmentContext().getAllocator();
- }
- return proxy;
- }
-
- public FragmentManager getFragmentManager(){
- return manager;
+ return allocator;
}
- final static String ERROR_MESSAGE = "Attempted to access AllocatorProxy";
-
- private static class AllocatorProxy implements BufferAllocator {
- private BufferAllocator allocator;
-
- public void setAllocator(BufferAllocator allocator) {
- this.allocator = allocator;
- }
-
- @Override
- public AccountingByteBuf buffer(int size) {
- if (allocator == null) {
- throw new UnsupportedOperationException(ERROR_MESSAGE);
- }
- return allocator.buffer(size);
- }
-
- @Override
- public AccountingByteBuf buffer(int minSize, int maxSize) {
- if (allocator == null) {
- throw new UnsupportedOperationException(ERROR_MESSAGE);
- }
- return allocator.buffer(minSize, maxSize);
- }
-
- @Override
- public ByteBufAllocator getUnderlyingAllocator() {
- if (allocator == null) {
- throw new UnsupportedOperationException(ERROR_MESSAGE);
- }
- return allocator.getUnderlyingAllocator();
- }
-
- @Override
- public BufferAllocator getChildAllocator(FragmentHandle handle, long initialReservation, long maximumReservation) throws OutOfMemoryException {
- if (allocator == null) {
- throw new UnsupportedOperationException(ERROR_MESSAGE);
- }
- return allocator.getChildAllocator(handle, initialReservation, maximumReservation);
- }
-
- @Override
- public PreAllocator getNewPreAllocator() {
- if (allocator == null) {
- throw new UnsupportedOperationException(ERROR_MESSAGE);
- }
- return allocator.getNewPreAllocator();
- }
-
- @Override
- public void close() {
- if (allocator != null) {
- allocator.close();
- }
- }
-
- @Override
- public long getAllocatedMemory() {
- if (allocator == null) {
- throw new UnsupportedOperationException(ERROR_MESSAGE);
- }
- return allocator.getAllocatedMemory();
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionCreator.java
index 9c2ef5b..197996d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionCreator.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.rpc.data;
import java.io.Closeable;
+import java.util.concurrent.ConcurrentMap;
import org.apache.drill.exec.exception.DrillbitStartupException;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
@@ -25,10 +26,11 @@ import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.rpc.control.WorkEventBus;
import org.apache.drill.exec.server.BootStrapContext;
+import com.google.common.collect.Maps;
import com.google.common.io.Closeables;
/**
- * Manages a connection pool for each endpoint.
+ * Manages a connection for each endpoint.
*/
public class DataConnectionCreator implements Closeable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DataConnectionCreator.class);
@@ -38,6 +40,7 @@ public class DataConnectionCreator implements Closeable {
private final WorkEventBus workBus;
private final DataResponseHandler dataHandler;
private final boolean allowPortHunting;
+ private ConcurrentMap<DrillbitEndpoint, DataConnectionManager> connectionManager = Maps.newConcurrentMap();
public DataConnectionCreator(BootStrapContext context, WorkEventBus workBus, DataResponseHandler dataHandler, boolean allowPortHunting) {
super();
@@ -55,7 +58,12 @@ public class DataConnectionCreator implements Closeable {
}
public DataTunnel getTunnel(DrillbitEndpoint endpoint, FragmentHandle handle) {
- return new DataTunnel(new DataConnectionManager(handle, endpoint, context));
+ DataConnectionManager newManager = new DataConnectionManager(handle, endpoint, context);
+ DataConnectionManager oldManager = connectionManager.putIfAbsent(endpoint, newManager);
+ if(oldManager != null){
+ newManager = oldManager;
+ }
+ return new DataTunnel(newManager);
}
public void close() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java
index aaff504..f1c94d3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java
@@ -22,10 +22,14 @@ import io.netty.buffer.ByteBuf;
import org.apache.drill.exec.proto.BitData.FragmentRecordBatch;
import org.apache.drill.exec.rpc.RemoteConnection;
import org.apache.drill.exec.rpc.Response;
+import org.apache.drill.exec.rpc.ResponseSender;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.work.fragment.FragmentManager;
public interface DataResponseHandler {
- public Response handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch, ByteBuf data) throws RpcException;
+ public void handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch,
+ ByteBuf data, ResponseSender responder) throws RpcException;
+
+ public void informOutOfMemory();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java
index b5b41a3..5d71c90 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java
@@ -26,6 +26,7 @@ import org.apache.drill.exec.record.RawFragmentBatch;
import org.apache.drill.exec.rpc.Acks;
import org.apache.drill.exec.rpc.RemoteConnection;
import org.apache.drill.exec.rpc.Response;
+import org.apache.drill.exec.rpc.ResponseSender;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.work.WorkManager.WorkerBee;
import org.apache.drill.exec.work.fragment.FragmentManager;
@@ -40,10 +41,17 @@ public class DataResponseHandlerImpl implements DataResponseHandler{
this.bee = bee;
}
- public Response handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch, ByteBuf data) throws RpcException {
+
+ @Override
+ public void informOutOfMemory() {
+ logger.error("Out of memory outside any particular fragment.");
+ }
+
+
+ public void handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch, ByteBuf data, ResponseSender sender) throws RpcException {
// logger.debug("Fragment Batch received {}", fragmentBatch);
try {
- boolean canRun = manager.handle(new RawFragmentBatch(connection, fragmentBatch, data));
+ boolean canRun = manager.handle(new RawFragmentBatch(connection, fragmentBatch, data, sender));
if (canRun) {
// logger.debug("Arriving batch means local batch can run, starting local batch.");
// if we've reached the canRun threshold, we'll proceed. This expects handler.handle() to only return a single
@@ -56,9 +64,8 @@ public class DataResponseHandlerImpl implements DataResponseHandler{
bee.getContext().getWorkBus().removeFragmentManager(manager.getHandle());
}
- return DataRpcConfig.OK;
} catch (FragmentSetupException e) {
logger.error("Failure while attempting to setup new fragment.", e);
- return new Response(RpcType.ACK, Acks.FAIL);
+ sender.send(new Response(RpcType.ACK, Acks.FAIL));
}
}}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
index ad57956..a449032 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.rpc.data;
+import io.netty.buffer.AccountingByteBuf;
import io.netty.buffer.ByteBuf;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
@@ -28,15 +29,19 @@ import org.apache.drill.exec.proto.BitData.BitClientHandshake;
import org.apache.drill.exec.proto.BitData.BitServerHandshake;
import org.apache.drill.exec.proto.BitData.FragmentRecordBatch;
import org.apache.drill.exec.proto.BitData.RpcType;
+import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.proto.UserBitShared.RpcChannel;
-import org.apache.drill.exec.record.RawFragmentBatch;
+import org.apache.drill.exec.proto.helper.QueryIdHelper;
+import org.apache.drill.exec.rpc.Acks;
import org.apache.drill.exec.rpc.BasicServer;
import org.apache.drill.exec.rpc.OutOfMemoryHandler;
import org.apache.drill.exec.rpc.ProtobufLengthDecoder;
import org.apache.drill.exec.rpc.Response;
+import org.apache.drill.exec.rpc.ResponseSender;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.control.WorkEventBus;
import org.apache.drill.exec.server.BootStrapContext;
+import org.apache.drill.exec.work.fragment.FragmentManager;
import com.google.protobuf.MessageLite;
@@ -47,7 +52,6 @@ public class DataServer extends BasicServer<RpcType, BitServerConnection> {
private final BootStrapContext context;
private final WorkEventBus workBus;
private final DataResponseHandler dataHandler;
- private BitServerConnection connection;
public DataServer(BootStrapContext context, WorkEventBus workBus, DataResponseHandler dataHandler) {
super(DataRpcConfig.MAPPING, context.getAllocator().getUnderlyingAllocator(), context.getBitLoopGroup());
@@ -69,7 +73,7 @@ public class DataServer extends BasicServer<RpcType, BitServerConnection> {
@Override
public BitServerConnection initRemoteConnection(Channel channel) {
- return connection = new BitServerConnection(channel, context.getAllocator());
+ return new BitServerConnection(channel, context.getAllocator());
}
@Override
@@ -86,20 +90,35 @@ public class DataServer extends BasicServer<RpcType, BitServerConnection> {
throw new RpcException(String.format("Invalid NodeMode. Expected BIT_DATA but received %s.",
inbound.getChannel()));
- connection.setManager(workBus.getOrCreateFragmentManager(inbound.getHandle()));
return BitServerHandshake.newBuilder().setRpcVersion(DataRpcConfig.RPC_VERSION).build();
}
};
}
+ private final static FragmentRecordBatch OOM_FRAGMENT = FragmentRecordBatch.newBuilder().setIsOutOfMemory(true).build();
@Override
- protected Response handle(BitServerConnection connection, int rpcType, ByteBuf pBody, ByteBuf body) throws RpcException {
+ protected void handle(BitServerConnection connection, int rpcType, ByteBuf pBody, ByteBuf body, ResponseSender sender) throws RpcException {
assert rpcType == RpcType.REQ_RECORD_BATCH_VALUE;
FragmentRecordBatch fragmentBatch = get(pBody, FragmentRecordBatch.PARSER);
- return dataHandler.handle(connection, connection.getFragmentManager(), fragmentBatch, body);
+ FragmentHandle handle = fragmentBatch.getHandle();
+
+ try {
+ FragmentManager manager = workBus.getOrCreateFragmentManager(fragmentBatch.getHandle());
+ BufferAllocator allocator = manager.getFragmentContext().getAllocator();
+ if(body != null){
+ if(!allocator.takeOwnership((AccountingByteBuf) body.unwrap())){
+ dataHandler.handle(connection, manager, OOM_FRAGMENT, null, null);
+ }
+ }
+ dataHandler.handle(connection, manager, fragmentBatch, body, sender);
+
+ } catch (FragmentSetupException e) {
+ logger.error("Failure while getting fragment manager. {}", QueryIdHelper.getQueryIdentifier(handle), e);
+ sender.send(new Response(RpcType.ACK, Acks.FAIL));
+ }
}
private class ProxyCloseHandler implements GenericFutureListener<ChannelFuture> {
@@ -111,14 +130,6 @@ public class DataServer extends BasicServer<RpcType, BitServerConnection> {
this.handler = handler;
}
- public GenericFutureListener<ChannelFuture> getHandler() {
- return handler;
- }
-
- public void setHandler(GenericFutureListener<ChannelFuture> handler) {
- this.handler = handler;
- }
-
@Override
public void operationComplete(ChannelFuture future) throws Exception {
handler.operationComplete(future);
@@ -126,20 +137,14 @@ public class DataServer extends BasicServer<RpcType, BitServerConnection> {
}
- private final static FragmentRecordBatch OOM_FRAGMENT = FragmentRecordBatch.newBuilder().setIsOutOfMemory(true).build();
+
@Override
public OutOfMemoryHandler getOutOfMemoryHandler() {
return new OutOfMemoryHandler() {
@Override
public void handle() {
- try {
- logger.debug("Setting autoRead false");
- connection.getFragmentManager().setAutoRead(false);
- connection.getFragmentManager().handle(new RawFragmentBatch(connection, OOM_FRAGMENT, null));
- } catch (FragmentSetupException e) {
- throw new RuntimeException();
- }
+ dataHandler.informOutOfMemory();
}
};
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataTunnel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataTunnel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataTunnel.java
index 3c2b9e3..546f967 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataTunnel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataTunnel.java
@@ -18,6 +18,9 @@
package org.apache.drill.exec.rpc.data;
import io.netty.buffer.ByteBuf;
+
+import java.util.concurrent.Semaphore;
+
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.proto.BitData.RpcType;
import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
@@ -25,13 +28,16 @@ import org.apache.drill.exec.record.FragmentWritableBatch;
import org.apache.drill.exec.rpc.DrillRpcFuture;
import org.apache.drill.exec.rpc.FutureBitCommand;
import org.apache.drill.exec.rpc.ListeningCommand;
+import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.RpcOutcomeListener;
+import org.apache.drill.exec.rpc.RpcConnectionHandler.FailureType;
public class DataTunnel {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DataTunnel.class);
private final DataConnectionManager manager;
+ private final Semaphore sendingSemaphore = new Semaphore(3);
public DataTunnel(DataConnectionManager manager) {
this.manager = manager;
@@ -39,18 +45,49 @@ public class DataTunnel {
public void sendRecordBatch(RpcOutcomeListener<Ack> outcomeListener, FragmentWritableBatch batch) {
SendBatchAsyncListen b = new SendBatchAsyncListen(outcomeListener, batch);
- manager.runCommand(b);
+ try{
+ sendingSemaphore.acquire();
+ manager.runCommand(b);
+ }catch(InterruptedException e){
+ outcomeListener.failed(new RpcException("Interrupted while trying to get sending semaphore.", e));
+ }
}
public DrillRpcFuture<Ack> sendRecordBatch(FragmentContext context, FragmentWritableBatch batch) {
SendBatchAsyncFuture b = new SendBatchAsyncFuture(batch, context);
- manager.runCommand(b);
+ try{
+ sendingSemaphore.acquire();
+ manager.runCommand(b);
+ }catch(InterruptedException e){
+ b.connectionFailed(FailureType.CONNECTION, new RpcException("Interrupted while trying to get sending semaphore.", e));
+ }
return b.getFuture();
}
-
- private static class SendBatchAsyncListen extends ListeningCommand<Ack, DataClientConnection> {
+ private class ThrottlingOutcomeListener implements RpcOutcomeListener<Ack>{
+ RpcOutcomeListener<Ack> inner;
+
+ public ThrottlingOutcomeListener(RpcOutcomeListener<Ack> inner) {
+ super();
+ this.inner = inner;
+ }
+
+ @Override
+ public void failed(RpcException ex) {
+ sendingSemaphore.release();
+ inner.failed(ex);
+ }
+
+ @Override
+ public void success(Ack value, ByteBuf buffer) {
+ sendingSemaphore.release();
+ inner.success(value, buffer);
+ }
+
+ }
+
+ private class SendBatchAsyncListen extends ListeningCommand<Ack, DataClientConnection> {
final FragmentWritableBatch batch;
public SendBatchAsyncListen(RpcOutcomeListener<Ack> listener, FragmentWritableBatch batch) {
@@ -60,7 +97,7 @@ public class DataTunnel {
@Override
public void doRpcCall(RpcOutcomeListener<Ack> outcomeListener, DataClientConnection connection) {
- connection.send(outcomeListener, RpcType.REQ_RECORD_BATCH, batch.getHeader(), Ack.class, batch.getBuffers());
+ connection.send(new ThrottlingOutcomeListener(outcomeListener), RpcType.REQ_RECORD_BATCH, batch.getHeader(), Ack.class, batch.getBuffers());
}
@Override
@@ -77,7 +114,7 @@ public class DataTunnel {
}
}
- private static class SendBatchAsyncFuture extends FutureBitCommand<Ack, DataClientConnection> {
+ private class SendBatchAsyncFuture extends FutureBitCommand<Ack, DataClientConnection> {
final FragmentWritableBatch batch;
final FragmentContext context;
@@ -89,7 +126,7 @@ public class DataTunnel {
@Override
public void doRpcCall(RpcOutcomeListener<Ack> outcomeListener, DataClientConnection connection) {
- connection.send(outcomeListener, RpcType.REQ_RECORD_BATCH, batch.getHeader(), Ack.class, batch.getBuffers());
+ connection.send(new ThrottlingOutcomeListener(outcomeListener), RpcType.REQ_RECORD_BATCH, batch.getHeader(), Ack.class, batch.getBuffers());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
index ddfb42c..41d13a0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.server;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import java.io.Closeable;
@@ -27,6 +29,7 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.metrics.DrillMetrics;
import org.apache.drill.exec.rpc.NamedThreadFactory;
+import org.apache.drill.exec.rpc.TransportCheck;
import com.codahale.metrics.MetricRegistry;
@@ -34,16 +37,16 @@ public class BootStrapContext implements Closeable{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BootStrapContext.class);
private final DrillConfig config;
- private final NioEventLoopGroup loop;
- private final NioEventLoopGroup loop2;
+ private final EventLoopGroup loop;
+ private final EventLoopGroup loop2;
private final MetricRegistry metrics;
private final BufferAllocator allocator;
public BootStrapContext(DrillConfig config) {
super();
this.config = config;
- this.loop = new NioEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS), new NamedThreadFactory("BitServer-"));
- this.loop2 = new NioEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS), new NamedThreadFactory("BitClient-"));
+ this.loop = TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS), "BitServer-");
+ this.loop2 = TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS), "BitClient-");
this.metrics = DrillMetrics.getInstance();
this.allocator = new TopLevelAllocator(config);
}
@@ -52,11 +55,11 @@ public class BootStrapContext implements Closeable{
return config;
}
- public NioEventLoopGroup getBitLoopGroup() {
+ public EventLoopGroup getBitLoopGroup() {
return loop;
}
- public NioEventLoopGroup getBitClientLoopGroup() {
+ public EventLoopGroup getBitClientLoopGroup() {
return loop2;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index 556724e..eed0126 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.server;
-import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.EventLoopGroup;
import java.util.Collection;
@@ -117,7 +117,7 @@ public class DrillbitContext {
return this.storagePlugins;
}
- public NioEventLoopGroup getBitLoopGroup(){
+ public EventLoopGroup getBitLoopGroup(){
return context.getBitLoopGroup();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index 73f97fe..01fcfe0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -17,12 +17,12 @@
*/
package org.apache.drill.exec.vector;
+import io.netty.buffer.AccountingByteBuf;
import io.netty.buffer.ByteBuf;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.exec.expr.holders.BitHolder;
import org.apache.drill.exec.expr.holders.NullableBitHolder;
-import org.apache.drill.exec.memory.AccountingByteBuf;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
@@ -366,11 +366,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
} else if (allocationMonitor > 0) {
allocationMonitor = 0;
}
- data.writerIndex(idx);
- if (data instanceof AccountingByteBuf) {
- data.capacity(idx);
- data.writerIndex(idx);
- }
+ VectorTrimmer.trim(data, idx);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
new file mode 100644
index 0000000..e946afe
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.AccountingByteBuf;
+import io.netty.buffer.ByteBuf;
+
+public class VectorTrimmer {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorTrimmer.class);
+
+ public static void trim(ByteBuf data, int idx) {
+ data.writerIndex(idx);
+ if (data instanceof AccountingByteBuf) {
+ // data.capacity(idx);
+ data.writerIndex(idx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
index a67f06b..23794d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
@@ -23,25 +23,23 @@ import java.lang.reflect.InvocationTargetException;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerArray;
-import java.util.concurrent.atomic.AtomicReferenceArray;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.base.Receiver;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.record.RawFragmentBatch;
-import org.apache.drill.exec.rpc.RemoteConnection;
+import org.apache.drill.exec.rpc.ResponseSender;
import com.google.common.base.Preconditions;
-public abstract class AbstractDataCollector implements DataCollector, ReadController{
+public abstract class AbstractDataCollector implements DataCollector{
private final List<DrillbitEndpoint> incoming;
private final int oppositeMajorFragmentId;
private final AtomicIntegerArray remainders;
private final AtomicInteger remainingRequired;
protected final RawBatchBuffer[] buffers;
- private final AtomicReferenceArray<RemoteConnection> connections;
private final AtomicInteger parentAccounter;
private final AtomicInteger finishedStreams = new AtomicInteger();
@@ -52,15 +50,14 @@ public abstract class AbstractDataCollector implements DataCollector, ReadContro
this.parentAccounter = parentAccounter;
this.incoming = receiver.getProvidingEndpoints();
- this.connections = new AtomicReferenceArray<>(incoming.size());
this.remainders = new AtomicIntegerArray(incoming.size());
this.oppositeMajorFragmentId = receiver.getOppositeMajorFragmentId();
this.buffers = new RawBatchBuffer[minInputsRequired];
try {
String bufferClassName = context.getConfig().getString(ExecConstants.INCOMING_BUFFER_IMPL);
- Constructor<?> bufferConstructor = Class.forName(bufferClassName).getConstructor(FragmentContext.class, ReadController.class, int.class);
+ Constructor<?> bufferConstructor = Class.forName(bufferClassName).getConstructor(FragmentContext.class, int.class);
for(int i = 0; i < buffers.length; i++) {
- buffers[i] = (RawBatchBuffer) bufferConstructor.newInstance(context, this, incoming.size());
+ buffers[i] = (RawBatchBuffer) bufferConstructor.newInstance(context, incoming.size());
}
} catch (InstantiationException | IllegalAccessException | InvocationTargetException |
NoSuchMethodException | ClassNotFoundException e) {
@@ -82,17 +79,6 @@ public abstract class AbstractDataCollector implements DataCollector, ReadContro
}
- public void setAutoRead(boolean enabled){
- for(int i = 0; i < connections.length(); i++){
- setAutoRead(i, enabled);
- }
- }
-
- public void setAutoRead(int minorFragmentId, boolean enabled){
- RemoteConnection c = connections.get(minorFragmentId);
- if(c != null) c.setAutoRead(enabled);
- }
-
public abstract void streamFinished(int minorFragmentId);
public boolean batchArrived(int minorFragmentId, RawFragmentBatch batch) throws IOException {
@@ -104,10 +90,6 @@ public abstract class AbstractDataCollector implements DataCollector, ReadContro
}
}
- // add the connection to the connection list if this is the first time we're seeing it.
- // TODO: move this to a better location (e.g. connection setup).
- if(connections.compareAndSet(minorFragmentId, null, batch.getConnection()));
-
// check to see if we have enough fragments reporting to proceed.
boolean decremented = false;
if (remainders.compareAndSet(minorFragmentId, 0, 1)) {
@@ -138,11 +120,6 @@ public abstract class AbstractDataCollector implements DataCollector, ReadContro
@Override
public void close() {
- for (int i = 0; i < connections.length(); i++) {
- if (connections.get(i) != null) {
- connections.get(i).close();
- };
- }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/DataCollector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/DataCollector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/DataCollector.java
index dc016be..42d8eae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/DataCollector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/DataCollector.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.work.batch;
import java.io.IOException;
import org.apache.drill.exec.record.RawFragmentBatch;
+import org.apache.drill.exec.rpc.ResponseSender;
interface DataCollector extends AutoCloseable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DataCollector.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java
index 3b97934..af5e9c7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java
@@ -28,6 +28,7 @@ import org.apache.drill.exec.physical.base.AbstractPhysicalVisitor;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.Receiver;
import org.apache.drill.exec.record.RawFragmentBatch;
+import org.apache.drill.exec.rpc.ResponseSender;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
@@ -81,10 +82,12 @@ public class IncomingBuffers implements AutoCloseable {
DataCollector fSet = fragCounts.get(sendMajorFragmentId);
if (fSet == null) throw new FragmentSetupException(String.format("We received a major fragment id that we were not expecting. The id was %d. %s", sendMajorFragmentId, Arrays.toString(fragCounts.values().toArray())));
try {
- boolean decremented = fSet.batchArrived(batch.getHeader().getSendingMinorFragmentId(), batch);
+ synchronized(this){
+ boolean decremented = fSet.batchArrived(batch.getHeader().getSendingMinorFragmentId(), batch);
- // we should only return true if remaining required has been decremented and is currently equal to zero.
- return decremented && remainingRequired.get() == 0;
+ // we should only return true if remaining required has been decremented and is currently equal to zero.
+ return decremented && remainingRequired.get() == 0;
+ }
} catch (IOException e) {
throw new FragmentSetupException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/RawBatchBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/RawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/RawBatchBuffer.java
index 23443fa..bc8a1dc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/RawBatchBuffer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/RawBatchBuffer.java
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.drill.exec.record.RawFragmentBatch;
import org.apache.drill.exec.record.RawFragmentBatchProvider;
-import org.apache.drill.exec.rpc.RemoteConnection;
+import org.apache.drill.exec.rpc.ResponseSender;
/**
* A batch buffer is responsible for queuing incoming batches until a consumer is ready to receive them. It will also
@@ -36,6 +36,7 @@ public interface RawBatchBuffer extends RawFragmentBatchProvider {
* @param batch
* Batch to enqueue
* @throws IOException
+ * @returns Whether response should be returned.
*/
public void enqueue(RawFragmentBatch batch) throws IOException;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/ReadController.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/ReadController.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/ReadController.java
deleted file mode 100644
index d23eb4c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/ReadController.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.work.batch;
-
-public interface ReadController {
- public void setAutoRead(boolean enabled);
- public void setAutoRead(int minorFragmentId, boolean enabled);
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/ResponseSenderQueue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/ResponseSenderQueue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/ResponseSenderQueue.java
new file mode 100644
index 0000000..5a9316e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/ResponseSenderQueue.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.work.batch;
+
+import java.util.Queue;
+
+import org.apache.drill.exec.rpc.ResponseSender;
+import org.apache.drill.exec.rpc.data.DataRpcConfig;
+
+import com.google.common.collect.Queues;
+
+public class ResponseSenderQueue {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ResponseSenderQueue.class);
+
+ private Queue<ResponseSender> q = Queues.newConcurrentLinkedQueue();
+
+ public void enqueueResponse(ResponseSender sender){
+ q.add(sender);
+ }
+
+ public void flushResponses(){
+ while(!q.isEmpty()){
+ ResponseSender s = q.poll();
+ if(s != null){
+ s.send(DataRpcConfig.OK);
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
index 9ee4d0f..76d642c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
@@ -37,7 +37,9 @@ import org.apache.drill.exec.proto.ExecProtos;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.record.RawFragmentBatch;
import org.apache.drill.exec.rpc.RemoteConnection;
+import org.apache.drill.exec.rpc.ResponseSender;
import org.apache.drill.exec.rpc.data.BitServerConnection;
+import org.apache.drill.exec.rpc.data.DataRpcConfig;
import org.apache.drill.exec.store.LocalSyncableFileSystem;
import org.apache.drill.exec.work.fragment.FragmentManager;
import org.apache.hadoop.conf.Configuration;
@@ -77,7 +79,7 @@ public class SpoolingRawBatchBuffer implements RawBatchBuffer {
private boolean closed = false;
private FragmentManager fragmentManager;
- public SpoolingRawBatchBuffer(FragmentContext context, ReadController readController, int fragmentCount) throws IOException, OutOfMemoryException {
+ public SpoolingRawBatchBuffer(FragmentContext context, int fragmentCount) throws IOException, OutOfMemoryException {
this.context = context;
this.allocator = context.getNewChildAllocator(ALLOCATOR_INITIAL_RESERVATION, ALLOCATOR_MAX_RESERVATION);
this.threshold = context.getConfig().getLong(ExecConstants.SPOOLING_BUFFER_MEMORY);
@@ -99,7 +101,8 @@ public class SpoolingRawBatchBuffer implements RawBatchBuffer {
public synchronized void enqueue(RawFragmentBatch batch) throws IOException {
if (batch.getHeader().getIsOutOfMemory()) {
if (fragmentManager == null) {
- fragmentManager = ((BitServerConnection) batch.getConnection()).getFragmentManager();
+ throw new UnsupportedOperationException("Need to fix.");
+// fragmentManager = ((BitServerConnection) batch.getConnection()).getFragmentManager();
}
// fragmentManager.setAutoRead(false);
// logger.debug("Setting autoRead false");
@@ -109,6 +112,7 @@ public class SpoolingRawBatchBuffer implements RawBatchBuffer {
} else {
logger.debug("ignoring duplicate OOM message");
}
+ batch.sendOk();
return;
}
RawFragmentBatchWrapper wrapper;
@@ -270,7 +274,7 @@ public class SpoolingRawBatchBuffer implements RawBatchBuffer {
BitData.FragmentRecordBatch header = BitData.FragmentRecordBatch.parseDelimitedFrom(stream);
ByteBuf buf = allocator.buffer(bodyLength);
buf.writeBytes(stream, bodyLength);
- batch = new RawFragmentBatch(null, header, buf);
+ batch = new RawFragmentBatch(null, header, buf, null);
buf.release();
available = true;
latch.countDown();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
index d14e50c..11e20c7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
@@ -23,6 +23,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.record.RawFragmentBatch;
+import org.apache.drill.exec.rpc.ResponseSender;
+import org.apache.drill.exec.rpc.data.DataRpcConfig;
import com.google.common.collect.Queues;
@@ -35,32 +37,20 @@ public class UnlimitedRawBatchBuffer implements RawBatchBuffer{
private final int startlimit;
private final AtomicBoolean overlimit = new AtomicBoolean(false);
private final AtomicBoolean outOfMemory = new AtomicBoolean(false);
- private final ReadController readController;
- private final boolean multiFragment;
+ private final ResponseSenderQueue readController = new ResponseSenderQueue();
- public UnlimitedRawBatchBuffer(FragmentContext context, ReadController readController, int fragmentCount) {
+ public UnlimitedRawBatchBuffer(FragmentContext context, int fragmentCount) {
int bufferSizePerSocket = context.getConfig().getInt(ExecConstants.INCOMING_BUFFER_SIZE);
- this.multiFragment = fragmentCount > 1;
- this.readController = readController;
this.softlimit = bufferSizePerSocket * fragmentCount;
this.startlimit = Math.max(softlimit/2, 1);
this.buffer = Queues.newLinkedBlockingDeque();
}
- private void setRead(int minorFragmentId, boolean setting){
- if(multiFragment){
- readController.setAutoRead(setting);
- }else{
- readController.setAutoRead(minorFragmentId, setting);
- }
- }
-
@Override
public void enqueue(RawFragmentBatch batch) {
if (batch.getHeader().getIsOutOfMemory()) {
logger.debug("Setting autoread false");
- readController.setAutoRead(false);
if (!outOfMemory.get() && !buffer.peekFirst().getHeader().getIsOutOfMemory()) {
buffer.addFirst(batch);
}
@@ -70,7 +60,9 @@ public class UnlimitedRawBatchBuffer implements RawBatchBuffer{
buffer.add(batch);
if(buffer.size() == softlimit){
overlimit.set(true);
- setRead(batch.getHeader().getSendingMinorFragmentId(), false);
+ readController.enqueueResponse(batch.getSender());
+ }else{
+ batch.sendOk();
}
}
@@ -98,7 +90,7 @@ public class UnlimitedRawBatchBuffer implements RawBatchBuffer{
if (outOfMemory.get() && buffer.size() < 10) {
logger.debug("Setting autoread true");
outOfMemory.set(false);
- readController.setAutoRead(true);
+ readController.flushResponses();
}
RawFragmentBatch b = null;
@@ -116,7 +108,6 @@ public class UnlimitedRawBatchBuffer implements RawBatchBuffer{
if (b != null && b.getHeader().getIsOutOfMemory()) {
outOfMemory.set(true);
- readController.setAutoRead(false);
return b;
}
@@ -125,8 +116,7 @@ public class UnlimitedRawBatchBuffer implements RawBatchBuffer{
if(!finished && overlimit.get()){
if(buffer.size() == startlimit){
overlimit.set(false);
- setRead(b.getHeader().getSendingMinorFragmentId(), true);
- readController.setAutoRead(true);
+ readController.flushResponses();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentManager.java
index 0a4b235..68849ff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentManager.java
@@ -22,6 +22,7 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.record.RawFragmentBatch;
import org.apache.drill.exec.rpc.RemoteConnection;
+import org.apache.drill.exec.rpc.ResponseSender;
/**
* The Fragment Manager is responsible managing incoming data and executing a fragment. Once enough data and resources
@@ -31,7 +32,7 @@ public interface FragmentManager {
/**
* Handle the next incoming record batch.
- *
+ *
* @param batch
* @return True if the fragment has enough incoming data to be able to be run.
* @throws FragmentSetupException
@@ -40,7 +41,7 @@ public interface FragmentManager {
/**
* Get the fragment runner for this incoming fragment. Note, this can only be requested once.
- *
+ *
* @return
*/
public abstract FragmentExecutor getRunnable();
@@ -50,7 +51,7 @@ public interface FragmentManager {
public boolean isWaiting();
public abstract FragmentHandle getHandle();
-
+
public abstract FragmentContext getFragmentContext();
public abstract void addConnection(RemoteConnection connection);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/NonRootFragmentManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/NonRootFragmentManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/NonRootFragmentManager.java
index 84fb806..51bf81c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/NonRootFragmentManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/NonRootFragmentManager.java
@@ -34,6 +34,7 @@ import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.record.RawFragmentBatch;
import org.apache.drill.exec.rpc.RemoteConnection;
+import org.apache.drill.exec.rpc.ResponseSender;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.work.batch.IncomingBuffers;
@@ -48,7 +49,6 @@ public class NonRootFragmentManager implements FragmentManager {
private volatile FragmentExecutor runner;
private volatile boolean cancel = false;
private final FragmentContext context;
- private final PhysicalPlanReader reader;
private List<RemoteConnection> connections = new CopyOnWriteArrayList<>();
public NonRootFragmentManager(PlanFragment fragment, DrillbitContext context) throws FragmentSetupException{
@@ -59,7 +59,6 @@ public class NonRootFragmentManager implements FragmentManager {
this.buffers = new IncomingBuffers(root, this.context);
this.context.setBuffers(buffers);
this.runnerListener = new NonRootStatusReporter(this.context, context.getController().getTunnel(fragment.getForeman()));
- this.reader = context.getPlanReader();
}catch(ExecutionSetupException | IOException e){
throw new FragmentSetupException("Failure while decoding fragment.", e);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/RootFragmentManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/RootFragmentManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/RootFragmentManager.java
index c763d55..4b9ee52 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/RootFragmentManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/RootFragmentManager.java
@@ -17,16 +17,17 @@
*/
package org.apache.drill.exec.work.fragment;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
import org.apache.drill.exec.exception.FragmentSetupException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.record.RawFragmentBatch;
import org.apache.drill.exec.rpc.RemoteConnection;
+import org.apache.drill.exec.rpc.ResponseSender;
import org.apache.drill.exec.work.batch.IncomingBuffers;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-
public class RootFragmentManager implements FragmentManager{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RootFragmentManager.class);
@@ -35,7 +36,7 @@ public class RootFragmentManager implements FragmentManager{
private final FragmentHandle handle;
private volatile boolean cancel = false;
private List<RemoteConnection> connections = new CopyOnWriteArrayList<>();
-
+
public RootFragmentManager(FragmentHandle handle, IncomingBuffers buffers, FragmentExecutor runner) {
super();
this.handle = handle;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java
index f579448..347cfb0 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java
@@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import mockit.Injectable;
+import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
@@ -34,8 +35,8 @@ import org.apache.drill.common.types.Types;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.proto.BitData.FragmentRecordBatch;
-import org.apache.drill.exec.proto.BitData.RpcType;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
@@ -43,14 +44,14 @@ import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.record.FragmentWritableBatch;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.WritableBatch;
-import org.apache.drill.exec.rpc.Acks;
import org.apache.drill.exec.rpc.RemoteConnection;
-import org.apache.drill.exec.rpc.Response;
+import org.apache.drill.exec.rpc.ResponseSender;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.RpcOutcomeListener;
import org.apache.drill.exec.rpc.control.WorkEventBus;
import org.apache.drill.exec.rpc.data.DataConnectionManager;
import org.apache.drill.exec.rpc.data.DataResponseHandler;
+import org.apache.drill.exec.rpc.data.DataRpcConfig;
import org.apache.drill.exec.rpc.data.DataServer;
import org.apache.drill.exec.rpc.data.DataTunnel;
import org.apache.drill.exec.vector.Float8Vector;
@@ -66,14 +67,26 @@ public class TestBitRpc extends ExecTest {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestBitRpc.class);
@Test
- public void testConnectionBackpressure(@Injectable WorkerBee bee, @Injectable WorkEventBus workBus) throws Exception {
- int port = 1234;
- BootStrapContext c = new BootStrapContext(DrillConfig.create());
+ public void testConnectionBackpressure(@Injectable WorkerBee bee, @Injectable final WorkEventBus workBus, @Injectable final FragmentManager fman, @Injectable final FragmentContext fcon) throws Exception {
+
+ final BootStrapContext c = new BootStrapContext(DrillConfig.create());
BootStrapContext c2 = new BootStrapContext(DrillConfig.create());
+
+ new NonStrictExpectations(){{
+ workBus.getOrCreateFragmentManager((FragmentHandle) any); result = fman;
+ workBus.getFragmentManager( (FragmentHandle) any); result = fman;
+ fman.getFragmentContext(); result = fcon;
+ fcon.getAllocator(); result = c.getAllocator();
+ }};
+
+
+ int port = 1234;
+
DataResponseHandler drp = new BitComTestHandler();
DataServer server = new DataServer(c, workBus, drp);
+
port = server.bind(port, false);
DrillbitEndpoint ep = DrillbitEndpoint.newBuilder().setAddress("localhost").setDataPort(port).build();
DataConnectionManager manager = new DataConnectionManager(FragmentHandle.getDefaultInstance(), ep, c2);
@@ -140,7 +153,7 @@ public class TestBitRpc extends ExecTest {
int v = 0;
@Override
- public Response handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch, ByteBuf data)
+ public void handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch, ByteBuf data, ResponseSender sender)
throws RpcException {
// System.out.println("Received.");
try {
@@ -153,7 +166,11 @@ public class TestBitRpc extends ExecTest {
// TODO Auto-generated catch block
e.printStackTrace();
}
- return new Response(RpcType.ACK, Acks.OK);
+ sender.send(DataRpcConfig.OK);
+ }
+
+ @Override
+ public void informOutOfMemory() {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/pom.xml b/exec/pom.xml
index 908d3bc..7248597 100644
--- a/exec/pom.xml
+++ b/exec/pom.xml
@@ -31,9 +31,7 @@
<dependencies>
</dependencies>
<modules>
- <module>bufferl</module>
<module>java-exec</module>
<module>jdbc</module>
- <!-- <module>jdbc-all</module> -->
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f43962a..7dbdef0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -395,7 +395,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
- <version>4.0.7.Final</version>
+ <version>4.0.20.Final</version>
</dependency>
<dependency>
[10/14] git commit: Small UI Fix: fix max batches and record counts
Posted by ja...@apache.org.
Small UI Fix: fix max batches and record counts
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/63d79a1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/63d79a1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/63d79a1d
Branch: refs/heads/master
Commit: 63d79a1d205c4be40cdf773e2b5c89869e284463
Parents: 208d511
Author: Steven Phillips <sp...@maprtech.com>
Authored: Sat Jun 21 22:38:14 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:10:12 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/server/rest/ProfileWrapper.java | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63d79a1d/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
index c6cd250..dcb7a45 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
@@ -119,10 +119,16 @@ public class ProfileWrapper {
ArrayList<OperatorProfile> ops = new ArrayList<OperatorProfile>(m.getOperatorProfileList());
long biggestIncomingRecords = 0;
long biggestBatches = 0;
-
- for (StreamProfile sp : ops.get(0).getInputProfileList()) {
- biggestIncomingRecords += sp.getRecords();
- biggestBatches += sp.getBatches();
+
+ for (OperatorProfile op : ops) {
+ long incomingRecords = 0;
+ long batches = 0;
+ for (StreamProfile sp : op.getInputProfileList()) {
+ incomingRecords += sp.getRecords();
+ batches += sp.getBatches();
+ }
+ biggestIncomingRecords = Math.max(biggestIncomingRecords, incomingRecords);
+ biggestBatches = Math.max(biggestBatches, batches);
}
builder.appendInteger(m.getMinorFragmentId(), null);
[12/14] git commit: Fix for sort ordering: canonicalize incoming
batches in top-n
Posted by ja...@apache.org.
Fix for sort ordering: canonicalize incoming batches in top-n
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0b296c82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0b296c82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0b296c82
Branch: refs/heads/master
Commit: 0b296c82e5e7b07de1266221f42761a590a476fa
Parents: 63d79a1
Author: Steven Phillips <sp...@maprtech.com>
Authored: Mon Jun 23 12:11:19 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:10:13 2014 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/physical/impl/TopN/TopNBatch.java | 7 +++++--
.../drill/exec/physical/impl/sort/RecordBatchData.java | 6 ++++++
.../exec/physical/impl/sort/SortRecordBatchBuilder.java | 8 +++++++-
exec/java-exec/src/main/resources/drill-module.conf | 2 +-
4 files changed, 19 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0b296c82/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
index 846d419..77be4ef 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
@@ -170,10 +170,12 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
case OK:
countSincePurge += incoming.getRecordCount();
batchCount++;
+ RecordBatchData batch = new RecordBatchData(incoming);
+ batch.canonicalize();
if (priorityQueue == null) {
- priorityQueue = createNewPriorityQueue(context, config.getOrderings(), new ExpandableHyperContainer(incoming), MAIN_MAPPING, LEFT_MAPPING, RIGHT_MAPPING);
+ priorityQueue = createNewPriorityQueue(context, config.getOrderings(), new ExpandableHyperContainer(batch.getContainer()), MAIN_MAPPING, LEFT_MAPPING, RIGHT_MAPPING);
}
- priorityQueue.add(context, new RecordBatchData(incoming));
+ priorityQueue.add(context, batch);
if (countSincePurge > config.getLimit() && batchCount > batchPurgeThreshold) {
purge();
countSincePurge = 0;
@@ -242,6 +244,7 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
selectionVector4.clear();
c.clear();
VectorContainer newQueue = new VectorContainer();
+ builder.canonicalize();
builder.build(context, newQueue);
priorityQueue.resetQueue(newQueue, builder.getSv4().createNewWrapperCurrent());
builder.getSv4().clear();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0b296c82/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/RecordBatchData.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/RecordBatchData.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/RecordBatchData.java
index 9cb6f79..02cad5a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/RecordBatchData.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/RecordBatchData.java
@@ -60,6 +60,12 @@ public class RecordBatchData {
container.setRecordCount(recordCount);
container.buildSchema(batch.getSchema().getSelectionVectorMode());
}
+
+ public void canonicalize() {
+ SelectionVectorMode mode = container.getSchema().getSelectionVectorMode();
+ container = VectorContainer.canonicalize(container);
+ container.buildSchema(mode);
+ }
public int getRecordCount(){
return recordCount;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0b296c82/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
index ba200f6..9626a97 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
@@ -112,6 +112,12 @@ public class SortRecordBatchBuilder {
return true;
}
+ public void canonicalize() {
+ for (RecordBatchData batch : batches.values()) {
+ batch.canonicalize();
+ }
+ }
+
public boolean isEmpty(){
return batches.isEmpty();
}
@@ -165,7 +171,7 @@ public class SortRecordBatchBuilder {
}
}
- for(MaterializedField f : vectors.keySet()){
+ for(MaterializedField f : schema){
List<ValueVector> v = vectors.get(f);
outputContainer.addHyperList(v, false);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0b296c82/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index 7f399b2..1519327 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -38,7 +38,7 @@ drill.exec: {
count: 7200,
delay: 500
},
- threads: 1
+ threads: 4
}
},
use.ip : false
[09/14] git commit: DRILL-1067: RPC layer improvements - Unfork Netty
and upgrade to 4.0.20.Final - Create support for EPOLL native impl on Linux
(disable by providing -Ddrill.exec.disable_linux_epoll=false) - Move BitData
connection to Multiplex with Da
Posted by ja...@apache.org.
DRILL-1067: RPC layer improvements
- Unfork Netty and upgrade to 4.0.20.Final
- Create support for EPOLL native impl on Linux (disable by providing -Ddrill.exec.disable_linux_epoll=false)
- Move BitData connection to Multiplex with DataTunnel level backpressure
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/af29c255
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/af29c255
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/af29c255
Branch: refs/heads/master
Commit: af29c2557b296b9bb33107edd02f8066950184f3
Parents: c3eea13
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Jun 21 17:59:56 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:09:50 2014 -0700
----------------------------------------------------------------------
distribution/pom.xml | 5 -
distribution/src/assemble/bin.xml | 1 -
exec/bufferl/pom.xml | 33 -
.../main/java/io/netty/buffer/PoolArenaL.java | 575 ------------
.../main/java/io/netty/buffer/PoolChunkL.java | 548 ------------
.../java/io/netty/buffer/PoolChunkListL.java | 212 -----
.../main/java/io/netty/buffer/PoolSubpageL.java | 207 -----
.../java/io/netty/buffer/PoolThreadCacheL.java | 34 -
.../netty/buffer/PooledByteBufAllocatorL.java | 287 ------
.../java/io/netty/buffer/PooledByteBufL.java | 229 -----
.../java/io/netty/buffer/PooledHeapBufferL.java | 299 -------
.../buffer/PooledUnsafeDirectByteBufL.java | 348 --------
.../main/java/io/netty/buffer/package-info.java | 73 --
.../java/io/netty/buffer/TestPoolChunkTrim.java | 218 -----
exec/java-exec/pom.xml | 16 +-
.../src/main/codegen/includes/vv_imports.ftl | 6 +-
.../codegen/templates/FixedValueVectors.java | 15 +-
.../templates/VariableLengthVectors.java | 5 +-
.../java/io/netty/buffer/AccountingByteBuf.java | 866 ++++++++++++++++++
.../netty/buffer/PooledByteBufAllocatorL.java | 85 ++
.../netty/buffer/UnsafeDirectLittleEndian.java | 179 ++++
.../apache/drill/exec/client/DrillClient.java | 6 +-
.../exec/coord/zk/ZKClusterCoordinator.java | 2 +-
.../drill/exec/memory/AccountingByteBuf.java | 867 -------------------
.../org/apache/drill/exec/memory/Accountor.java | 20 +-
.../drill/exec/memory/AtomicRemainder.java | 15 +-
.../drill/exec/memory/BufferAllocator.java | 15 +-
.../drill/exec/memory/TopLevelAllocator.java | 21 +-
.../drill/exec/proto/helper/QueryIdHelper.java | 15 +-
.../drill/exec/record/RawFragmentBatch.java | 17 +-
.../org/apache/drill/exec/rpc/BasicClient.java | 11 +-
.../org/apache/drill/exec/rpc/BasicServer.java | 16 +-
.../apache/drill/exec/rpc/ResponseSender.java | 24 +
.../java/org/apache/drill/exec/rpc/RpcBus.java | 46 +-
.../apache/drill/exec/rpc/TransportCheck.java | 78 ++
.../exec/rpc/data/BitServerConnection.java | 94 +-
.../exec/rpc/data/DataConnectionCreator.java | 12 +-
.../exec/rpc/data/DataResponseHandler.java | 6 +-
.../exec/rpc/data/DataResponseHandlerImpl.java | 15 +-
.../apache/drill/exec/rpc/data/DataServer.java | 49 +-
.../apache/drill/exec/rpc/data/DataTunnel.java | 51 +-
.../drill/exec/server/BootStrapContext.java | 15 +-
.../drill/exec/server/DrillbitContext.java | 4 +-
.../org/apache/drill/exec/vector/BitVector.java | 8 +-
.../apache/drill/exec/vector/VectorTrimmer.java | 33 +
.../exec/work/batch/AbstractDataCollector.java | 31 +-
.../drill/exec/work/batch/DataCollector.java | 1 +
.../drill/exec/work/batch/IncomingBuffers.java | 9 +-
.../drill/exec/work/batch/RawBatchBuffer.java | 3 +-
.../drill/exec/work/batch/ReadController.java | 23 -
.../exec/work/batch/ResponseSenderQueue.java | 45 +
.../exec/work/batch/SpoolingRawBatchBuffer.java | 10 +-
.../work/batch/UnlimitedRawBatchBuffer.java | 28 +-
.../exec/work/fragment/FragmentManager.java | 7 +-
.../work/fragment/NonRootFragmentManager.java | 3 +-
.../exec/work/fragment/RootFragmentManager.java | 9 +-
.../apache/drill/exec/server/TestBitRpc.java | 33 +-
exec/pom.xml | 2 -
pom.xml | 2 +-
59 files changed, 1639 insertions(+), 4248 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index b395a7b..ce051bc 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -41,11 +41,6 @@
</dependency>
<dependency>
<groupId>org.apache.drill.exec</groupId>
- <artifactId>drill-buffers</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.drill.exec</groupId>
<artifactId>drill-java-exec</artifactId>
<version>${project.version}</version>
<classifier>rebuffed</classifier>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index 65bc6e0..bc8fc5f 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -42,7 +42,6 @@
<!-- Now, select which projects to include in this module-set. -->
<includes>
<include>org.apache.drill.exec:drill-jdbc:jar</include>
- <include>org.apache.drill.exec:drill-buffers</include>
<include>org.apache.drill:drill-protocol:jar:rebuffed</include>
<include>org.apache.drill:drill-common:jar:rebuffed</include>
<include>org.apache.drill.exec:drill-java-exec:jar:rebuffed</include>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/pom.xml
----------------------------------------------------------------------
diff --git a/exec/bufferl/pom.xml b/exec/bufferl/pom.xml
deleted file mode 100644
index a8e48ad..0000000
--- a/exec/bufferl/pom.xml
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- ~ Copyright 2012 The Netty Project ~ ~ The Netty Project licenses this
- file to you under the Apache License, ~ version 2.0 (the "License"); you
- may not use this file except in compliance ~ with the License. You may obtain
- a copy of the License at: ~ ~ http://www.apache.org/licenses/LICENSE-2.0
- ~ ~ Unless required by applicable law or agreed to in writing, software ~
- distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- ~ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- ~ License for the specific language governing permissions and limitations
- ~ under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.drill.exec</groupId>
- <artifactId>exec-parent</artifactId>
- <version>1.0.0-m2-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>drill-buffers</artifactId>
-
- <name>exec/Netty Little Endian Buffers</name>
-
- <dependencies>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-buffer</artifactId>
- <version>4.0.7.Final</version>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/main/java/io/netty/buffer/PoolArenaL.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/main/java/io/netty/buffer/PoolArenaL.java b/exec/bufferl/src/main/java/io/netty/buffer/PoolArenaL.java
deleted file mode 100644
index 479fa80..0000000
--- a/exec/bufferl/src/main/java/io/netty/buffer/PoolArenaL.java
+++ /dev/null
@@ -1,575 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import io.netty.util.internal.PlatformDependent;
-import io.netty.util.internal.StringUtil;
-
-import java.nio.ByteBuffer;
-
-
-/**
- * A PoolArenaL is a single arena for managing memory.
- * It represents a composite memory manager which consists of other
- * memory allocators, where each of the allocators
- * works best for different allocation sizes.
- *
- * More specifically, a PoolArenaL includes the following allocators:
- * tiny: (16-256) - multiple lists of pages of items, where each list has items of a fixed size 16*N.
- * small: (512-pageSize/2) - multiple lists of pages of items, where each list has items of a fixed size 2^N.
- * normal: (pageSize-chunkSize) - a skiplist of chunks ordered to minimize fragmentation, where
- * each chunk is divided into pages using the buddy system.
- * huge: (>chunkSize) - memory is allocated directly from the operating system.
- *
- * Note: In a multi-threaded environment, the "parent" creates multiple "arenas",
- * distributing threads among them to minimize contention.
- *
- * @param <T>
- */
-abstract class PoolArenaL<T> {
-
- final PooledByteBufAllocatorL parent;
-
- private final int pageSize;
- private final int maxOrder;
- private final int pageShifts;
- private final int chunkSize;
- private final int subpageOverflowMask;
-
- private final PoolSubpageL<T>[] tinySubpagePools;
- private final PoolSubpageL<T>[] smallSubpagePools;
-
- private final PoolChunkListL<T> q050;
- private final PoolChunkListL<T> q025;
- private final PoolChunkListL<T> q000;
- private final PoolChunkListL<T> qInit;
- private final PoolChunkListL<T> q075;
- private final PoolChunkListL<T> q100;
-
- // TODO: Test if adding padding helps under contention
- //private long pad0, pad1, pad2, pad3, pad4, pad5, pad6, pad7;
-
- /**
- * Create a new arena.
- * @param parent - The global memory manager (where we go to allocate a completely new buffer).
- * @param pageSize - The minimum size of memory for using the "normal" allocation (buddy system).
- * @param maxOrder - The size of a "chunk", where chunkSize = pageSize * 2^maxOrder.
- * @param pageShifts - pageSize expressed as a power of 2. (redundant?)
- * @param chunkSize - chunkSize in bytes. (redundant?)
- */
- protected PoolArenaL(PooledByteBufAllocatorL parent, int pageSize, int maxOrder, int pageShifts, int chunkSize) {
- this.parent = parent;
- this.pageSize = pageSize;
- this.maxOrder = maxOrder;
- this.pageShifts = pageShifts;
- this.chunkSize = chunkSize;
- subpageOverflowMask = ~(pageSize - 1);
-
- // Create the tiny pools, ranging in size from 16 to 256 in steps of 16.
- tinySubpagePools = newSubpagePoolArray(512 >>> 4);
- for (int i = 0; i < tinySubpagePools.length; i ++) {
- tinySubpagePools[i] = newSubpagePoolHead(pageSize);
- }
-
- // Create the small pools, ranging in size from 512 to pagesize/2 as powers of 2.
- smallSubpagePools = newSubpagePoolArray(pageShifts - 9);
- for (int i = 0; i < smallSubpagePools.length; i ++) {
- smallSubpagePools[i] = newSubpagePoolHead(pageSize);
- }
-
- // Create a skip list of chunks consisting of separate lists connected together.
- // Chunks migrate between the lists depending on how much free space they have.
- q100 = new PoolChunkListL<T>(this, null, 100, Integer.MAX_VALUE);
- q075 = new PoolChunkListL<T>(this, q100, 75, 100);
- q050 = new PoolChunkListL<T>(this, q075, 50, 100);
- q025 = new PoolChunkListL<T>(this, q050, 25, 75);
- q000 = new PoolChunkListL<T>(this, q025, 1, 50);
- qInit = new PoolChunkListL<T>(this, q000, Integer.MIN_VALUE, 25);
- q100.prevList = q075;
- q075.prevList = q050;
- q050.prevList = q025;
- q025.prevList = q000;
- q000.prevList = null;
- qInit.prevList = qInit;
- }
-
- /** Initialize a subpage list */
- private PoolSubpageL<T> newSubpagePoolHead(int pageSize) {
- PoolSubpageL<T> head = new PoolSubpageL<T>(pageSize);
- head.prev = head;
- head.next = head;
- return head;
- }
-
- /** Create an array (uninitialized) of subpage lists.*/
- @SuppressWarnings("unchecked")
- private PoolSubpageL<T>[] newSubpagePoolArray(int size) {
- return new PoolSubpageL[size];
- }
-
-
- /**
- * Allocate a buffer from the current arena.
- * Instead of being an upper bounds sanity check,
- * the "max" capacity is used to opportunistically allocate extra memory.
- * Later, the capacity can be reduced very efficiently.
- *
- * @param cache TODO: not sure
- * @param minRequested The smallest capacity buffer we want
- * @param maxRequested If convenient, allocate up to this capacity
- * @return A buffer with capacity between min and max capacity
- */
- PooledByteBufL<T> allocate(PoolThreadCacheL cache, int minRequested, int maxRequested) {
-
- // Create a buffer header, limiting growth to minimize copying
- PooledByteBufL<T> buf = newByteBuf(Integer.MAX_VALUE);
- allocate(cache, buf, minRequested, maxRequested);
- return buf;
- }
-
- /**
- * Allocate memory to a buffer container.
- * @param cache TODO: not sure
- * @param buf - A buffer which will contain the allocated memory
- * @param minRequested - The smallest amount of memory.
- * @param maxRequested - The maximum memory to allocate if convenient.
- */
- private void allocate(PoolThreadCacheL cache, PooledByteBufL<T> buf, final int minRequested, int maxRequested) {
- // This code should be reorganized.
- // case: <= maxTiny: allocateTiny{select which tiny list, allocate subpage from list.}
- // case: <= maxSmall: allocateSmall{select which small list, allocate subpage from list.}
- // case: <= maxNormal: allocateNormal
- // otherwise: allocateHuge
- // where maxTiny=256, maxSmall=pageSize/2, maxNormal=chunkSize.
-
- // CASE: minCapacity is a subpage
- final int normCapacity = normalizeCapacity(minRequested);
- if ((normCapacity & subpageOverflowMask) == 0) { // minCapacity <= pageSize/2
- int tableIdx;
- PoolSubpageL<T>[] table;
-
- // if "tiny", pick list based on multiple of 16
- if ((normCapacity & 0xFFFFFE00) == 0) { // minCapacity < 512
- tableIdx = normCapacity >>> 4;
- table = tinySubpagePools;
-
- // else "small", pick list based on power of 2.
- } else {
- tableIdx = 0;
- int i = normCapacity >>> 10;
- while (i != 0) {
- i >>>= 1;
- tableIdx ++;
- }
- table = smallSubpagePools;
- }
-
- // Whether tiny or small, allocate an item from the first page in the corresponding list
- synchronized (this) {
- final PoolSubpageL<T> head = table[tableIdx];
- final PoolSubpageL<T> s = head.next;
- if (s != head) {
- assert s.doNotDestroy && s.elemSize == normCapacity;
- long handle = s.allocate();
- assert handle >= 0;
- s.chunk.initBufWithSubpage(buf, handle, minRequested);
- return;
- }
- }
-
- // If the list was empty, allocate a new page, allocate an item, and add page to the list.
- // This is awkward. "allocateNormal" does subpage allocation internally,
- // and it really shouldn't know anything at all about subpages.
- // Instead, we should allocate a complete page and
- // add it to the desired list ourselves.
- allocateNormal(buf, minRequested, normCapacity);
- return;
-
- // CASE: HUGE allocation.
- } else if (normCapacity > chunkSize) {
- allocateHuge(buf, minRequested);
- return;
- }
-
- // OTHERWISE: Normal allocation of pages from a chunk.
- allocateNormal(buf, minRequested, maxRequested);
- }
-
-
- /**
- * Allocate a "normal" (page .. chunk) sized buffer from a chunk in the skiplist.
- * @param buf - the buffer header which will receive the memory.
- * @param minRequested - the minimum requested capacity in bytes.
- * @param maxRequested - the maximum requested capacity.
- */
- private synchronized void allocateNormal(PooledByteBufL<T> buf, int minRequested, int maxRequested) {
-
- // If the buffer can be allocated from the skip list, then allocate it.
- if (q050.allocate(buf, minRequested, maxRequested) || q025.allocate(buf, minRequested, maxRequested) ||
- q000.allocate(buf, minRequested, maxRequested) || qInit.allocate(buf, minRequested, maxRequested) ||
- q075.allocate(buf, minRequested, maxRequested) || q100.allocate(buf, minRequested, maxRequested)) {
- return;
- }
-
- // Create a new chunk.
- PoolChunkL<T> c = newChunk(pageSize, maxOrder, pageShifts, chunkSize);
-
- // Allocate a buffer from the chunk.
- long handle = c.allocate(minRequested, maxRequested);
- assert handle > 0;
- c.initBuf(buf, handle, minRequested, maxRequested);
-
- // Add the new chunk to the skip list at the "newly initialized" location.
- qInit.add(c);
- }
-
-
- /**
- * Allocate a huge (>chunksize) buffer.
- * @param buf
- * @param reqCapacity
- */
- private void allocateHuge(PooledByteBufL<T> buf, int reqCapacity) {
- buf.initUnpooled(newUnpooledChunk(reqCapacity), reqCapacity);
- }
-
-
- /**
- * Free a piece of memory.
- * @param chunk
- * @param handle
- */
- synchronized void free(PoolChunkL<T> chunk, long handle) {
- if (chunk.unpooled) {
- destroyChunk(chunk);
- } else {
- chunk.parent.free(chunk, handle);
- }
- }
-
-
-
- /**
- * Find which list holds subpage buffers of the given size.
- * @param elemSize
- * @return
- */
- PoolSubpageL<T> findSubpagePoolHead(int elemSize) {
- int tableIdx;
- PoolSubpageL<T>[] table;
- if ((elemSize & 0xFFFFFE00) == 0) { // < 512
- tableIdx = elemSize >>> 4;
- table = tinySubpagePools;
- } else {
- tableIdx = 0;
- elemSize >>>= 10;
- while (elemSize != 0) {
- elemSize >>>= 1;
- tableIdx ++;
- }
- table = smallSubpagePools;
- }
-
- return table[tableIdx];
- }
-
-
-
- /**
- * Bump the requested size up to the size which will actually be allocated
- * @param reqCapacity - the requested size
- * @return the large, normalized size
- */
- private int normalizeCapacity(int reqCapacity) {
- if (reqCapacity < 0) {
- throw new IllegalArgumentException("capacity: " + reqCapacity + " (expected: 0+)");
- }
-
- // CASE: HUGE allocation, don't change it.
- if (reqCapacity >= chunkSize) {
- return reqCapacity;
- }
-
- // CASE: normal or small allocation, then round up to 2^n
- if ((reqCapacity & 0xFFFFFE00) != 0) { // >= 512
- // Doubled
-
- int normalizedCapacity = reqCapacity;
- normalizedCapacity |= normalizedCapacity >>> 1;
- normalizedCapacity |= normalizedCapacity >>> 2;
- normalizedCapacity |= normalizedCapacity >>> 4;
- normalizedCapacity |= normalizedCapacity >>> 8;
- normalizedCapacity |= normalizedCapacity >>> 16;
- normalizedCapacity ++;
-
- if (normalizedCapacity < 0) {
- normalizedCapacity >>>= 1;
- }
-
- return normalizedCapacity;
- }
-
- // OTHERWISE: tiny allocations. Round up to the next multiple of 16
- if ((reqCapacity & 15) == 0) {
- return reqCapacity;
- }
-
- return (reqCapacity & ~15) + 16;
- }
-
-
- /**
- * Change the size of a buffer by allocating new memory and copying the old data to it.
- * @param buf - the buffer containing the memory
- * @param newCapacity - the desired capacity
- * @param freeOldMemory - whether to release the old memory or not.
- */
- void reallocate(PooledByteBufL<T> buf, int newCapacity, boolean freeOldMemory) {
-
- // Sanity check to not grow beyond the maxCapacity.
- // This check may not be relevant any more since we have reinterpreted maxCapacity.
- if (newCapacity < 0 || newCapacity > buf.maxCapacity()) {
- throw new IllegalArgumentException("newCapacity: " + newCapacity);
- }
-
- // Do nothing if capacity doesn't actually change.
- int oldCapacity = buf.length;
- if (oldCapacity == newCapacity) {
- return;
- }
-
- // Cache some local values to make them more accessible.
- PoolChunkL<T> oldChunk = buf.chunk;
- long oldHandle = buf.handle;
- T oldMemory = buf.memory;
- int oldOffset = buf.offset;
- int readerIndex = buf.readerIndex();
- int writerIndex = buf.writerIndex();
-
- // Allocate new memory for the buffer
- allocate(parent.threadCache.get(), buf, newCapacity, newCapacity);
-
- // CASE: buffer has grown. Copy data from old buffer to new.
- if (newCapacity > oldCapacity) {
- memoryCopy(
- oldMemory, oldOffset + readerIndex,
- buf.memory, buf.offset + readerIndex, writerIndex - readerIndex);
-
- // CASE: buffer has shrunk. Copy data, but also reset the reader/writer positions.
- } else if (newCapacity < oldCapacity) {
- if (readerIndex < newCapacity) {
- if (writerIndex > newCapacity) {
- writerIndex = newCapacity;
- }
- memoryCopy(
- oldMemory, oldOffset + readerIndex,
- buf.memory, buf.offset + readerIndex, writerIndex - readerIndex);
- } else {
- readerIndex = writerIndex = newCapacity;
- }
- }
-
- buf.setIndex(readerIndex, writerIndex); // move to buffer has shrunk case?
-
- // If requested, release the old memory.
- if (freeOldMemory) {
- free(oldChunk, oldHandle);
- }
- }
-
- /** Create a chunkSize chunk of memory which will be part of the pool */
- protected abstract PoolChunkL<T> newChunk(int pageSize, int maxOrder, int pageShifts, int chunkSize);
-
- /** Create an arbitrary size chunk of memory which is not part of the pool. */
- protected abstract PoolChunkL<T> newUnpooledChunk(int capacity);
-
- /** Create a new buffer, but don't allocate memory yet. */
- protected abstract PooledByteBufL<T> newByteBuf(int maxCapacity);
-
- /** Copy memory from one allocation to another. */
- protected abstract void memoryCopy(T src, int srcOffset, T dst, int dstOffset, int length);
-
- /** Release or recycle a chunk of memory */
- protected abstract void destroyChunk(PoolChunkL<T> chunk);
-
- /** Display the contents of a PoolArena */
- public synchronized String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append("Chunk(s) at 0~25%:");
- buf.append(StringUtil.NEWLINE);
- buf.append(qInit);
- buf.append(StringUtil.NEWLINE);
- buf.append("Chunk(s) at 0~50%:");
- buf.append(StringUtil.NEWLINE);
- buf.append(q000);
- buf.append(StringUtil.NEWLINE);
- buf.append("Chunk(s) at 25~75%:");
- buf.append(StringUtil.NEWLINE);
- buf.append(q025);
- buf.append(StringUtil.NEWLINE);
- buf.append("Chunk(s) at 50~100%:");
- buf.append(StringUtil.NEWLINE);
- buf.append(q050);
- buf.append(StringUtil.NEWLINE);
- buf.append("Chunk(s) at 75~100%:");
- buf.append(StringUtil.NEWLINE);
- buf.append(q075);
- buf.append(StringUtil.NEWLINE);
- buf.append("Chunk(s) at 100%:");
- buf.append(StringUtil.NEWLINE);
- buf.append(q100);
- buf.append(StringUtil.NEWLINE);
- buf.append("tiny subpages:");
- for (int i = 1; i < tinySubpagePools.length; i ++) {
- PoolSubpageL<T> head = tinySubpagePools[i];
- if (head.next == head) {
- continue;
- }
-
- buf.append(StringUtil.NEWLINE);
- buf.append(i);
- buf.append(": ");
- PoolSubpageL<T> s = head.next;
- for (;;) {
- buf.append(s);
- s = s.next;
- if (s == head) {
- break;
- }
- }
- }
- buf.append(StringUtil.NEWLINE);
- buf.append("small subpages:");
- for (int i = 0; i < smallSubpagePools.length; i ++) {
- PoolSubpageL<T> head = smallSubpagePools[i];
- if (head.next == head) {
- continue;
- }
-
- buf.append(StringUtil.NEWLINE);
- buf.append(i);
- buf.append(": ");
- PoolSubpageL<T> s = head.next;
- for (;;) {
- buf.append(s);
- s = s.next;
- if (s == head) {
- break;
- }
- }
- }
- buf.append(StringUtil.NEWLINE);
-
- return buf.toString();
- }
-
-
- /** HeapArena is an arena which allocates memory from the Java Heap */
- static final class HeapArena extends PoolArenaL<byte[]> {
-
- HeapArena(PooledByteBufAllocatorL parent, int pageSize, int maxOrder, int pageShifts, int chunkSize) {
- super(parent, pageSize, maxOrder, pageShifts, chunkSize);
- }
-
- @Override
- protected PoolChunkL<byte[]> newChunk(int pageSize, int maxOrder, int pageShifts, int chunkSize) {
- return new PoolChunkL<byte[]>(this, new byte[chunkSize], pageSize, maxOrder, pageShifts, chunkSize);
- }
-
- @Override
- protected PoolChunkL<byte[]> newUnpooledChunk(int capacity) {
- return new PoolChunkL<byte[]>(this, new byte[capacity], capacity);
- }
-
- @Override
- protected void destroyChunk(PoolChunkL<byte[]> chunk) {
- // Rely on GC.
- }
-
- @Override
- protected PooledByteBufL<byte[]> newByteBuf(int maxCapacity) {
- return PooledHeapByteBufL.newInstance(maxCapacity);
- }
-
- @Override
- protected void memoryCopy(byte[] src, int srcOffset, byte[] dst, int dstOffset, int length) {
- if (length == 0) {
- return;
- }
-
- System.arraycopy(src, srcOffset, dst, dstOffset, length);
- }
- }
-
-
- /** DirectArena is an arena which allocates memory from off-heap (direct allocation).*/
- static final class DirectArena extends PoolArenaL<ByteBuffer> {
-
- private static final boolean HAS_UNSAFE = PlatformDependent.hasUnsafe();
-
- DirectArena(PooledByteBufAllocatorL parent, int pageSize, int maxOrder, int pageShifts, int chunkSize) {
- super(parent, pageSize, maxOrder, pageShifts, chunkSize);
- }
-
- @Override
- protected PoolChunkL<ByteBuffer> newChunk(int pageSize, int maxOrder, int pageShifts, int chunkSize) {
- return new PoolChunkL<ByteBuffer>(
- this, ByteBuffer.allocateDirect(chunkSize), pageSize, maxOrder, pageShifts, chunkSize);
- }
-
- @Override
- protected PoolChunkL<ByteBuffer> newUnpooledChunk(int capacity) {
- return new PoolChunkL<ByteBuffer>(this, ByteBuffer.allocateDirect(capacity), capacity);
- }
-
- @Override
- protected void destroyChunk(PoolChunkL<ByteBuffer> chunk) {
- PlatformDependent.freeDirectBuffer(chunk.memory);
- }
-
- @Override
- protected PooledByteBufL<ByteBuffer> newByteBuf(int maxCapacity) {
- if (HAS_UNSAFE) {
- return PooledUnsafeDirectByteBufL.newInstance(maxCapacity);
- } else {
- throw new UnsupportedOperationException();
-// return PooledDirectByteBufL.newInstance(maxCapacity);
- }
- }
-
- @Override
- protected void memoryCopy(ByteBuffer src, int srcOffset, ByteBuffer dst, int dstOffset, int length) {
- if (length == 0) {
- return;
- }
-
- if (HAS_UNSAFE) {
- PlatformDependent.copyMemory(
- PlatformDependent.directBufferAddress(src) + srcOffset,
- PlatformDependent.directBufferAddress(dst) + dstOffset, length);
- } else {
- // We must duplicate the NIO buffers because they may be accessed by other Netty buffers.
- src = src.duplicate();
- dst = dst.duplicate();
- src.position(srcOffset).limit(srcOffset + length);
- dst.position(dstOffset);
- dst.put(src);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/main/java/io/netty/buffer/PoolChunkL.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/main/java/io/netty/buffer/PoolChunkL.java b/exec/bufferl/src/main/java/io/netty/buffer/PoolChunkL.java
deleted file mode 100644
index 724e25f..0000000
--- a/exec/bufferl/src/main/java/io/netty/buffer/PoolChunkL.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-
-/**
- * A Chunk is a large, fixed size piece of memory allocated from the operating system.
- * This PoolChunk allocator divides a chunk into a run of pages using the buddy system.
- * The actual allocation will be the size requested, rounded up to the next
- * power-of-2.
- *
- * This allocator does "normal" allocations, where the requested size varies
- * from page size to chunk size.
- *
- * The allocator is based on buddy system. It views a chunk as a binary tree with
- * 1 run of chunksize, or
- * 2 runs of chunksize/2, or
- * 4 runs of chunksize/4, or
- * ...
- * 2^maxOrder runs of pagesize
- *
- * Each node in the binary tree is labeled:
- * - unused. The node and all its children are unallocated
- * - allocated. The node (and all its children) are allocated
- * as a single request. The children remain marked "unused".
- * - branch. At least one descendent is allocated.
- *
- * The binary tree is represented in the "memoryMap",
- * which saves the node status in a simple array without using links.
- * The array indices indicate the position within the tree as follows:
- * 0 - unused
- * 1 - root
- * 2,3 - children of 1
- * 4,5 6,7 - children of 2 and 3
- * 8,9 10,11 12,13 14,15 - next level of children.
- *
- * Thus, i/2 points to the parent of i,
- * i*2 points to left child, i*2+1 points to right child.
- *
- * Note the current code also deals with smaller subpage allocations.
- * The overall memory manager only comes here when it wants a new page,
- * not every time it allocates a subpage piece of memory.
- */
-final class PoolChunkL<T> {
- private static final int ST_UNUSED = 0;
- private static final int ST_BRANCH = 1;
- private static final int ST_ALLOCATED = 2;
- private static final int ST_ALLOCATED_SUBPAGE = ST_ALLOCATED | 1;
-
- private static final long multiplier = 0x5DEECE66DL;
- private static final long addend = 0xBL;
- private static final long mask = (1L << 48) - 1;
-
- final PoolArenaL<T> arena;
- final T memory;
- final boolean unpooled;
-
- private final int[] memoryMap;
- private final PoolSubpageL<T>[] subpages;
- /** Used to determine if the requested capacity is equal to or greater than pageSize. */
- private final int subpageOverflowMask;
- private final int pageSize;
- private final int pageShifts;
-
- private final int chunkSize;
- private final int maxSubpageAllocs;
-
- private long random = (System.nanoTime() ^ multiplier) & mask;
-
- private int freeBytes;
-
- PoolChunkListL<T> parent;
- PoolChunkL<T> prev;
- PoolChunkL<T> next;
-
- // TODO: Test if adding padding helps under contention
- //private long pad0, pad1, pad2, pad3, pad4, pad5, pad6, pad7;
-
- /** Create a new "chunk" of memory to be used in the given arena.
- *
- * @param arena - the arena this chunk belongs to
- * @param memory
- * @param pageSize - the size of a page (known to arena, why here?)
- * @param maxOrder - how many pages to a chunk (2^^maxOrder pages)
- * @param pageShifts - page size as number of shifts (2^^pageShifts)
- * @param chunkSize - the size of a chunk (pagesize*2^^maxOrder(known to arena)
- */
- PoolChunkL(PoolArenaL<T> arena, T memory, int pageSize, int maxOrder, int pageShifts, int chunkSize) {
- unpooled = false;
- this.arena = arena;
- this.memory = memory;
- this.pageSize = pageSize;
- this.pageShifts = pageShifts;
- this.chunkSize = chunkSize;
- subpageOverflowMask = ~(pageSize - 1);
- freeBytes = chunkSize;
-
- int chunkSizeInPages = chunkSize >>> pageShifts;
- maxSubpageAllocs = 1 << maxOrder;
-
- // Generate the memory map.
- memoryMap = new int[maxSubpageAllocs << 1];
- int memoryMapIndex = 1;
- for (int i = 0; i <= maxOrder; i ++) {
- int runSizeInPages = chunkSizeInPages >>> i;
- for (int j = 0; j < chunkSizeInPages; j += runSizeInPages) {
- //noinspection PointlessBitwiseExpression
- memoryMap[memoryMapIndex ++] = j << 17 | runSizeInPages << 2 | ST_UNUSED;
- }
- }
-
- subpages = newSubpageArray(maxSubpageAllocs);
- }
-
- /** Creates a special chunk that is not pooled. */
- PoolChunkL(PoolArenaL<T> arena, T memory, int size) {
- unpooled = true;
- this.arena = arena;
- this.memory = memory;
- memoryMap = null;
- subpages = null;
- subpageOverflowMask = 0;
- pageSize = 0;
- pageShifts = 0;
- chunkSize = size;
- maxSubpageAllocs = 0;
- }
-
- /** Creates an uninitialized array of subpage lists */
- @SuppressWarnings("unchecked")
- private PoolSubpageL<T>[] newSubpageArray(int size) {
- return new PoolSubpageL[size];
- }
-
-
- /** returns the percentage of the chunk which has been allocated */
- int usage() {
- if (freeBytes == 0) {
- return 100;
- }
-
- int freePercentage = (int) (freeBytes * 100L / chunkSize);
- if (freePercentage == 0) {
- return 99;
- }
- return 100 - freePercentage;
- }
-
-
- /**
- * Allocates a buffer of the given size from current chunk
- * @param capacity - requested capacity of the buffer
- * @return - handle to the buffer, -1 if failed
- */
- long allocate(int capacity) {
- return allocate(capacity, capacity);
- }
-
-
- /**
- * Allocates a buffer with size between minRequested and maxRequested
- * @param minRequested
- * @param maxRequested
- * @return
- */
- long allocate(int minRequested, int maxRequested) {
-
- // CASE: allocating runs of pages, make use of maxCapacity since we can trim it later
- if (maxRequested > pageSize/2) {
- return allocateRun(minRequested, maxRequested, 1, chunkSize);
- }
-
- // OTHERWISE: allocating subpage buffer. Special case: maxCapacity is normCapacity.
- // Note: this case should be moved to PoolArena.
- else {
- return allocateSubpage(maxRequested, 1, memoryMap[1]);
- }
- }
-
-
-
- /**
- * Allocate a run of pages where the run size is within minCapacity thru maxCapacity.
- * @param minRequested - the minimum size of the buffer
- * @param maxRequested - the maximum size of the buffer to be allocated if convenient
- * @param node - the subtree of this chunk to search
- * @return handle to the allocated memory
- *
- * More specifically, this routine finds an unused node in the binary tree,
- * s.t. the node is big enough to contain minCapacity, and is not
- * bigger than the size to contain maxCapacity.
- *
- * A node is the correct size to contain x bytes, if
- * runlength(node) == roundup-power-of-2(x)
- * equivalently, runlength(node) >= x and runlength(node.child) < x
- * equivalently, runlength(node) >= x and runlength(node)/2 < x
- *
- * Similarly, a node is the correct size to contain min...max bytes, if
- * runlength(node) >= roundup-power-of-2(min) && runlength(node) <= roundup-power-of-2(max)
- * or equivalently, runlength(node) >= min && runlength(node)/2 < max
- */
- long allocateRun(int minRequested, int maxRequested, int node, int runLength) {
-
- // Search through the subtrees until finding an unused node s.t. runlength >= min
- for (; runLength >= minRequested; runLength /= 2) {
- if ((memoryMap[node]&3) != ST_BRANCH) break;
-
- // Search one random subtree (recursively)
- int child = node*2 + nextRandom();
- long handle = allocateRun(minRequested, maxRequested, child, runLength/2);
- if (handle != -1) return handle;
-
- // If not found, search the other subtree (tail recursion)
- node = child ^ 1;
- }
-
- // if we failed to find an unused node which is big enough, then failure.
- if (runLength < minRequested || (memoryMap[node]&3) != ST_UNUSED) {
- return -1;
- }
-
- // At this point, we have an unused node s.t. runlength >= min.
- // In other words, it is larger than the minimum, but it may also be larger
- // than the maximum.
-
- // Continue descending subtree looking for a node s.t. runlength/2 < max
- for (; runLength/2 >= maxRequested; runLength/=2) {
-
- // We are about to allocate from one of our children, so we become BRANCH
- memoryMap[node] = (memoryMap[node]&~3) | ST_BRANCH;
-
- // Pick one of the children and continue descending its subtree.
- node = node * 2 + nextRandom();
- }
-
- // We are at an unused node which satisfies both conditions. Allocate it.
- memoryMap[node] = (memoryMap[node]&~3) | ST_ALLOCATED;
- freeBytes -= runLength;
- return node;
- }
-
-
-
-
-
- /**
- * Allocate a new page for splitting into subpage items.
- * Note: this routine doesn't belong here. Instead, we should have a "subpage" allocator
- * which is invoked instead of us and is responsible for the entire subpage allocation sizes.
- * @param normCapacity - the actual size of the buffer we will allocate
- * @param curIdxn - the node where our search stargs
- * @param val - contents of the current node
- * @return a handle to the allocated buffer.
- */
- private long allocateSubpage(int normCapacity, int curIdx, int val) {
- int state = val & 3;
- if (state == ST_BRANCH) {
- int nextIdx = curIdx << 1 ^ nextRandom();
- long res = branchSubpage(normCapacity, nextIdx);
- if (res > 0) {
- return res;
- }
-
- return branchSubpage(normCapacity, nextIdx ^ 1);
- }
-
- if (state == ST_UNUSED) {
- return allocateSubpageSimple(normCapacity, curIdx, val);
- }
-
- if (state == ST_ALLOCATED_SUBPAGE) {
- PoolSubpageL<T> subpage = subpages[subpageIdx(curIdx)];
- int elemSize = subpage.elemSize;
- if (normCapacity != elemSize) {
- return -1;
- }
-
- return subpage.allocate();
- }
-
- return -1;
- }
-
-
-
- /**
- * Allocate a page to be used for subpage buffers, knowing the subtree is UNUSED
- * @param normCapacity
- * @param curIdx
- * @param val
- * @return
- */
- private long allocateSubpageSimple(int normCapacity, int curIdx, int val) {
- int runLength = runLength(val);
- for (;;) {
- if (runLength == pageSize) {
- memoryMap[curIdx] = val & ~3 | ST_ALLOCATED_SUBPAGE;
- freeBytes -= runLength;
-
- int subpageIdx = subpageIdx(curIdx);
- PoolSubpageL<T> subpage = subpages[subpageIdx];
- if (subpage == null) {
- subpage = new PoolSubpageL<T>(this, curIdx, runOffset(val), pageSize, normCapacity);
- subpages[subpageIdx] = subpage;
- } else {
- subpage.init(normCapacity);
- }
- return subpage.allocate();
- }
-
- int nextIdx = curIdx << 1 ^ nextRandom();
- int unusedIdx = nextIdx ^ 1;
-
- memoryMap[curIdx] = val & ~3 | ST_BRANCH;
- //noinspection PointlessBitwiseExpression
- memoryMap[unusedIdx] = memoryMap[unusedIdx] & ~3 | ST_UNUSED;
-
- runLength >>>= 1;
- curIdx = nextIdx;
- val = memoryMap[curIdx];
- }
- }
-
- private long branchSubpage(int normCapacity, int nextIdx) {
- int nextVal = memoryMap[nextIdx];
- if ((nextVal & 3) != ST_ALLOCATED) {
- return allocateSubpage(normCapacity, nextIdx, nextVal);
- }
- return -1;
- }
-
- /**
- * Free up memory to reduce the size of a run of pages.
- * The resulting run starts on the same page, and
- * the trailing pages are returned to the memory manager.
- * Trim is intended to be an efficient way to reduce the size of a buffer.
- * No new memory is allocated, nor is any data copied.
- * @param handle - which run of pages was allocated from the current chunk
- * @param smallerSize - the new desired size the new run of pages
- * @return a new handle to the smaller run of pages, or -1 if can't trim.
- */
- long trim(long handle, int smallerSize) {
- int memoryMapIdx = (int) handle;
- int bitmapIx = (int)(handle >>> 32);
- int originalRunLength = runLength(memoryMap[memoryMapIdx]);
-
- // If the buffer was a HUGE allocation, then we leave it alone
- if (this.unpooled || memoryMapIdx == 0 || handle < 0) {
- return -1;
- }
-
- // If the buffer was a subpage, then we also leave it alone.
- if (bitmapIx != 0 || (memoryMap[memoryMapIdx] & 3) == ST_ALLOCATED_SUBPAGE) {
- return -1;
- }
-
- // We can't trim if the result will become a subpage
- if (smallerSize <= pageSize/2) {
- return -1;
- }
-
- // If the buffer is growing, then we aren't really trimming.
- if (smallerSize >= originalRunLength) {
- return -1;
- }
-
- // Starting at current node, follow left hand children, until reaching node of desired size.
- // Note that runLength and memoryMapIdx move in unison.
- int runLength;
- for (runLength = originalRunLength; smallerSize*2 <= runLength; runLength /= 2, memoryMapIdx = memoryMapIdx<<1) {
-
- // Current node is a parent of the desired node. It now becomes a "BRANCH".
- memoryMap[memoryMapIdx] = (memoryMap[memoryMapIdx] & ~3) | ST_BRANCH;
-
- // Right hand child is now an unused buddy. Mark it "UNUSED".
- // (done - should already be marked "UNUSED")
- }
-
- // We are now at the desired node. Mark it allocated.
- memoryMap[memoryMapIdx] = (memoryMap[memoryMapIdx] & ~3) | ST_ALLOCATED;
- freeBytes += (originalRunLength - runLength);
-
- // return new handle to the reduced size buffer.
- return memoryMapIdx;
- }
-
- /**
- * Return a buffer back to the memory pool.
- * @param handle
- */
- void free(long handle) {
- int memoryMapIdx = (int) handle;
- int bitmapIdx = (int) (handle >>> 32);
-
- // If buffer was allocated from a subpage, then free it.
- int val = memoryMap[memoryMapIdx];
- int state = val & 3;
- if (state == ST_ALLOCATED_SUBPAGE) {
- assert bitmapIdx != 0;
- PoolSubpageL<T> subpage = subpages[subpageIdx(memoryMapIdx)];
- assert subpage != null && subpage.doNotDestroy;
- if (subpage.free(bitmapIdx & 0x3FFFFFFF)) {
- return;
- }
-
- // Otherwise, it should have been allocated from the chunk
- } else {
- assert state == ST_ALLOCATED : "state: " + state;
- assert bitmapIdx == 0;
- }
-
- // Update the nr of bytes free
- freeBytes += runLength(val);
-
- // start at current node and work up the tree
- for (;;) {
-
- // Mark the node as "unused"
- memoryMap[memoryMapIdx] = val & ~3 | ST_UNUSED;
-
- // If at top of tree, done
- if (memoryMapIdx == 1) {
- assert freeBytes == chunkSize;
- return;
- }
-
- // If the buddy is allocated, we can stop since no more merging can occur
- if ((memoryMap[siblingIdx(memoryMapIdx)] & 3) != ST_UNUSED) {
- break;
- }
-
- // move to current node's parent, effectively merging with UNUSED buddy
- memoryMapIdx = parentIdx(memoryMapIdx);
- val = memoryMap[memoryMapIdx];
- }
- }
-
- void initBuf(PooledByteBufL<T> buf, long handle, int reqCapacity) {
- initBuf(buf, handle, reqCapacity, reqCapacity);
- }
-
- /**
- * Initialize a buffer given a handle that was allocated from this chunk.
- * @param buf The buffer to be initialized.
- * @param handle The handle representing memory allocated from this chunk.
- * @param minRequested The minimum requested capacity.
- * @param maxRequested The maximum requested capacity.
- */
- void initBuf(PooledByteBufL<T> buf, long handle, int minRequested, int maxRequested) {
- int memoryMapIdx = (int) handle;
- int bitmapIdx = (int) (handle >>> 32);
-
- // If this is a normal allocation
- if (bitmapIdx == 0) {
-
- // Verify the memory is allocated
- int val = memoryMap[memoryMapIdx];
- assert (val & 3) == ST_ALLOCATED : String.valueOf(val & 3);
-
- // Initialize buffer with as large as possible capacity within the requested range
- // (assert: we know the buffer is >= minCapacity)
- int capacity = Math.min(runLength(val), maxRequested);
- buf.init(this, handle, runOffset(val), capacity, runLength(val));
-
- // Otherwise initialize buffer as a subpage allocation.
- } else {
- initBufWithSubpage(buf, handle, bitmapIdx, minRequested);
- }
- }
-
- void initBufWithSubpage(PooledByteBufL<T> buf, long handle, int reqCapacity) {
- initBufWithSubpage(buf, handle, (int) (handle >>> 32), reqCapacity);
- }
-
- private void initBufWithSubpage(PooledByteBufL<T> buf, long handle, int bitmapIdx, int reqCapacity) {
- assert bitmapIdx != 0;
-
- int memoryMapIdx = (int) handle;
- int val = memoryMap[memoryMapIdx];
- assert (val & 3) == ST_ALLOCATED_SUBPAGE;
-
- PoolSubpageL<T> subpage = subpages[subpageIdx(memoryMapIdx)];
- assert subpage.doNotDestroy;
- assert reqCapacity <= subpage.elemSize;
-
- buf.init(
- this, handle,
- runOffset(val) + (bitmapIdx & 0x3FFFFFFF) * subpage.elemSize, reqCapacity, subpage.elemSize);
- }
-
- private static int parentIdx(int memoryMapIdx) {
- return memoryMapIdx >>> 1;
- }
-
- private static int siblingIdx(int memoryMapIdx) {
- return memoryMapIdx ^ 1;
- }
-
- private int runLength(int val) {
- return (val >>> 2 & 0x7FFF) << pageShifts;
- }
-
- private int runOffset(int val) {
- return val >>> 17 << pageShifts;
- }
-
- private int subpageIdx(int memoryMapIdx) {
- return memoryMapIdx - maxSubpageAllocs;
- }
-
- private int nextRandom() {
- random = random * multiplier + addend & mask;
- return (int) (random >>> 47) & 1;
- }
-
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append("Chunk(");
- buf.append(Integer.toHexString(System.identityHashCode(this)));
- buf.append(": ");
- buf.append(usage());
- buf.append("%, ");
- buf.append(chunkSize - freeBytes);
- buf.append('/');
- buf.append(chunkSize);
- buf.append(')');
- return buf.toString();
- }
-
- public int getPageSize() {
- return pageSize;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/main/java/io/netty/buffer/PoolChunkListL.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/main/java/io/netty/buffer/PoolChunkListL.java b/exec/bufferl/src/main/java/io/netty/buffer/PoolChunkListL.java
deleted file mode 100644
index 7e25557..0000000
--- a/exec/bufferl/src/main/java/io/netty/buffer/PoolChunkListL.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import io.netty.util.internal.StringUtil;
-
-
-/**
- * A list of chunks with similar "usage". If a chunk is added to the wrong list,
- * it will migrate to the next list which hopefully will be the correct one.
- *
- * @param <T>
- */
-final class PoolChunkListL<T> {
- private final PoolArenaL<T> arena;
- private final PoolChunkListL<T> nextList;
- PoolChunkListL<T> prevList;
-
- private final int minUsage;
- private final int maxUsage;
-
- private PoolChunkL<T> head;
-
- // TODO: Test if adding padding helps under contention
- //private long pad0, pad1, pad2, pad3, pad4, pad5, pad6, pad7;
-
- /**
- * Create a new, empty pool of chunks.
- * @param arena - the bigger arena this pool belongs to
- * @param nextList - the next list to consider (in the same pool)
- * @param minUsage - contains chunks with the specified usage (min ... max)
- * @param maxUsage
- */
- PoolChunkListL(PoolArenaL<T> arena, PoolChunkListL<T> nextList, int minUsage, int maxUsage) {
- this.arena = arena;
- this.nextList = nextList;
- this.minUsage = minUsage;
- this.maxUsage = maxUsage;
- }
-
- /**
- * Allocate a buffer with the requested size
- * @param buf - the container to hold the buffer
- * @param minRequested - the minumum requested capacity
- * @param maxRequested - the maximum capacity
- * @return
- */
- boolean allocate(PooledByteBufL<T> buf, int minRequested, int maxRequested) {
-
- // If list is empty, then allocation fails
- if (head == null) {
- return false;
- }
-
- // Do for each chunk in the list
- for (PoolChunkL<T> cur = head;;) {
-
- // If we successfully allocated from the chunk ...
- long handle = cur.allocate(minRequested, maxRequested);
- if (handle < 0) {
- cur = cur.next;
- if (cur == null) {
- return false;
- }
-
- // ... then add the memory to the buffer container
- } else {
- cur.initBuf(buf, handle, minRequested, maxRequested);
-
- // If usage changed, then move to next list
- if (cur.usage() >= maxUsage) {
- remove(cur);
- nextList.add(cur);
- }
- return true;
- }
- }
- }
-
-
- /**
- * Release a buffer back to the original chunk.
- * @param chunk
- * @param handle
- */
- void free(PoolChunkL<T> chunk, long handle) {
-
- // Release buffer back to the original chunk
- chunk.free(handle);
-
- // If usage changed, then move to different list
- if (chunk.usage() < minUsage) {
- remove(chunk);
- if (prevList == null) {
- assert chunk.usage() == 0;
- arena.destroyChunk(chunk);
- } else {
- prevList.add(chunk);
- }
- }
- }
-
- /**
- * Shrink the buffer down to the specified size, freeing up unused memory.
- * @param chunk - chunk the buffer resides in
- * @param handle - the handle to the buffer
- * @param size - the new desired "size"
- * @return a new handle to the smaller buffer
- */
- long trim(PoolChunkL<T> chunk, long handle, int size) {
- synchronized(arena){
- // Trim the buffer, possibly getting a new handle.
- handle = chunk.trim(handle, size);
- if (handle == -1) return handle;
-
- // Move the chunk to a different list if usage changed significantly
- if (chunk.usage() < minUsage) {
- assert chunk.usage() > 0 && prevList != null;
- remove(chunk);
- prevList.add(chunk);
- }
-
- // return new handle for the smaller buffer
- return handle;
- }
- }
-
-
- /**
- * Add a chunk to the current chunklist
- * @param chunk
- */
- void add(PoolChunkL<T> chunk) {
-
- // If usage has change, then add to the neighboring list instead
- int usage = chunk.usage();
- if (usage >= maxUsage) {
- nextList.add(chunk);
- return;
- //} else if (usage < minUsage) { // TODO: Could this result in a recursive loop?
- // prevList.add(chunk);
- // return;
- }
-
- // Add chunk to linked list.
- chunk.parent = this;
- if (head == null) {
- head = chunk;
- chunk.prev = null;
- chunk.next = null;
- } else {
- chunk.prev = null;
- chunk.next = head;
- head.prev = chunk;
- head = chunk;
- }
- }
-
-
- /**
- * Remove a chunk from the current linked list of chunks
- * @param cur
- */
- private void remove(PoolChunkL<T> cur) {
- if (cur == head) {
- head = cur.next;
- if (head != null) {
- head.prev = null;
- }
- } else {
- PoolChunkL<T> next = cur.next;
- cur.prev.next = next;
- if (next != null) {
- next.prev = cur.prev;
- }
- }
- }
-
- @Override
- public String toString() {
- if (head == null) {
- return "none";
- }
-
- StringBuilder buf = new StringBuilder();
- for (PoolChunkL<T> cur = head;;) {
- buf.append(cur);
- cur = cur.next;
- if (cur == null) {
- break;
- }
- buf.append(StringUtil.NEWLINE);
- }
-
- return buf.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/main/java/io/netty/buffer/PoolSubpageL.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/main/java/io/netty/buffer/PoolSubpageL.java b/exec/bufferl/src/main/java/io/netty/buffer/PoolSubpageL.java
deleted file mode 100644
index 41051d1..0000000
--- a/exec/bufferl/src/main/java/io/netty/buffer/PoolSubpageL.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-final class PoolSubpageL<T> {
-
- final PoolChunkL<T> chunk;
- final int memoryMapIdx;
- final int runOffset;
- final int pageSize;
- final long[] bitmap;
-
- PoolSubpageL<T> prev;
- PoolSubpageL<T> next;
-
- boolean doNotDestroy;
- int elemSize;
- int maxNumElems;
- int nextAvail;
- int bitmapLength;
- int numAvail;
-
- // TODO: Test if adding padding helps under contention
- //private long pad0, pad1, pad2, pad3, pad4, pad5, pad6, pad7;
-
- /** Special constructor that creates a linked list head */
- PoolSubpageL(int pageSize) {
- chunk = null;
- memoryMapIdx = -1;
- runOffset = -1;
- elemSize = -1;
- this.pageSize = pageSize;
- bitmap = null;
- }
-
- PoolSubpageL(PoolChunkL<T> chunk, int memoryMapIdx, int runOffset, int pageSize, int elemSize) {
- this.chunk = chunk;
- this.memoryMapIdx = memoryMapIdx;
- this.runOffset = runOffset;
- this.pageSize = pageSize;
- bitmap = new long[pageSize >>> 10]; // pageSize / 16 / 64
- init(elemSize);
- }
-
- void init(int elemSize) {
- doNotDestroy = true;
- this.elemSize = elemSize;
- if (elemSize != 0) {
- maxNumElems = numAvail = pageSize / elemSize;
- nextAvail = 0;
- bitmapLength = maxNumElems >>> 6;
- if ((maxNumElems & 63) != 0) {
- bitmapLength ++;
- }
-
- for (int i = 0; i < bitmapLength; i ++) {
- bitmap[i] = 0;
- }
- }
-
- addToPool();
- }
-
- /**
- * Allocates a subpage element from a page.
- * @return the bitmap index of the subpage allocation.
- */
- long allocate() {
- if (elemSize == 0) {
- return toHandle(0);
- }
-
- if (numAvail == 0 || !doNotDestroy) {
- return -1;
- }
-
- final int bitmapIdx = nextAvail;
- int q = bitmapIdx >>> 6;
- int r = bitmapIdx & 63;
- assert (bitmap[q] >>> r & 1) == 0;
- bitmap[q] |= 1L << r;
-
- if (-- numAvail == 0) {
- removeFromPool();
- nextAvail = -1;
- } else {
- nextAvail = findNextAvailable();
- }
-
- return toHandle(bitmapIdx);
- }
-
- /**
- * Frees an element and returns it to the page it came from.
- * @return {@code true} if this subpage is in use.
- * {@code false} if this subpage is not used by its chunk and thus it's OK to be released.
- */
- boolean free(int bitmapIdx) {
-
- if (elemSize == 0) {
- return true;
- }
-
- // mark the corresponding bit as "free"
- int q = bitmapIdx >>> 6;
- int r = bitmapIdx & 63;
- assert (bitmap[q] >>> r & 1) != 0;
- bitmap[q] ^= 1L << r;
-
- // If we were full, add our page to the pool of subpages
- if (numAvail ++ == 0) {
- nextAvail = bitmapIdx;
- addToPool();
- return true;
- }
-
- // If we are not empty, then keep us among the pool of subpages
- if (numAvail != maxNumElems) {
- return true;
-
- // if our page is now empty (numAvail == maxNumElems), ...
- } else {
- // Do not remove if this page is the only one left in the pool.
- if (prev == next) {
- return true;
- }
-
- // Remove this subpage from the pool if there are other subpages left in the pool.
- doNotDestroy = false;
- removeFromPool();
- return false;
- }
- }
-
- private void addToPool() {
- PoolSubpageL<T> head = chunk.arena.findSubpagePoolHead(elemSize);
- assert prev == null && next == null;
- prev = head;
- next = head.next;
- next.prev = this;
- head.next = this;
- }
-
- private void removeFromPool() {
- assert prev != null && next != null;
- prev.next = next;
- next.prev = prev;
- next = null;
- prev = null;
- }
-
-
- /**
- * Search a bitmap to find the next available bit.
- * @return index to a free bit
- */
- private int findNextAvailable() {
- int newNextAvail = -1;
- loop:
- for (int i = 0; i < bitmapLength; i ++) {
- long bits = bitmap[i];
- if (~bits != 0) {
- for (int j = 0; j < 64; j ++) {
- if ((bits & 1) == 0) {
- newNextAvail = i << 6 | j;
- break loop;
- }
- bits >>>= 1;
- }
- }
- }
-
- if (newNextAvail < maxNumElems) {
- return newNextAvail;
- } else {
- return -1;
- }
- }
-
- private long toHandle(int bitmapIdx) {
- return 0x4000000000000000L | (long) bitmapIdx << 32 | memoryMapIdx;
- }
-
- public String toString() {
- if (!doNotDestroy) {
- return "(" + memoryMapIdx + ": not in use)";
- }
-
- return String.valueOf('(') + memoryMapIdx + ": " + (maxNumElems - numAvail) + '/' + maxNumElems +
- ", offset: " + runOffset + ", length: " + pageSize + ", elemSize: " + elemSize + ')';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/main/java/io/netty/buffer/PoolThreadCacheL.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/main/java/io/netty/buffer/PoolThreadCacheL.java b/exec/bufferl/src/main/java/io/netty/buffer/PoolThreadCacheL.java
deleted file mode 100644
index 8045db3..0000000
--- a/exec/bufferl/src/main/java/io/netty/buffer/PoolThreadCacheL.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import java.nio.ByteBuffer;
-
-final class PoolThreadCacheL {
-
- final PoolArenaL<byte[]> heapArena;
- final PoolArenaL<ByteBuffer> directArena;
-
- // TODO: Test if adding padding helps under contention
- //private long pad0, pad1, pad2, pad3, pad4, pad5, pad6, pad7;
-
- PoolThreadCacheL(PoolArenaL<byte[]> heapArena, PoolArenaL<ByteBuffer> directArena) {
- this.heapArena = heapArena;
- this.directArena = directArena;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/bufferl/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
deleted file mode 100644
index bc2b137..0000000
--- a/exec/bufferl/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import io.netty.buffer.AbstractByteBufAllocator;
-import io.netty.buffer.ByteBuf;
-import io.netty.util.internal.PlatformDependent;
-import io.netty.util.internal.StringUtil;
-import io.netty.util.internal.SystemPropertyUtil;
-import io.netty.util.internal.logging.InternalLogger;
-import io.netty.util.internal.logging.InternalLoggerFactory;
-
-import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicInteger;
-
-public class PooledByteBufAllocatorL extends AbstractByteBufAllocator {
-
- private static final InternalLogger logger = InternalLoggerFactory.getInstance(PooledByteBufAllocatorL.class);
-
- private static final int DEFAULT_NUM_HEAP_ARENA;
- private static final int DEFAULT_NUM_DIRECT_ARENA;
-
- private static final int DEFAULT_PAGE_SIZE;
- private static final int DEFAULT_MAX_ORDER; // 8192 << 11 = 16 MiB per chunk
-
- private static final int MIN_PAGE_SIZE = 4096;
- private static final int MAX_CHUNK_SIZE = (int) (((long) Integer.MAX_VALUE + 1) / 2);
-
- static {
- int defaultPageSize = SystemPropertyUtil.getInt("io.netty.allocator.pageSize", 8192);
- Throwable pageSizeFallbackCause = null;
- try {
- validateAndCalculatePageShifts(defaultPageSize);
- } catch (Throwable t) {
- pageSizeFallbackCause = t;
- defaultPageSize = 8192;
- }
- DEFAULT_PAGE_SIZE = defaultPageSize;
-
- int defaultMaxOrder = SystemPropertyUtil.getInt("io.netty.allocator.maxOrder", 11);
- Throwable maxOrderFallbackCause = null;
- try {
- validateAndCalculateChunkSize(DEFAULT_PAGE_SIZE, defaultMaxOrder);
- } catch (Throwable t) {
- maxOrderFallbackCause = t;
- defaultMaxOrder = 11;
- }
- DEFAULT_MAX_ORDER = defaultMaxOrder;
-
- // Determine reasonable default for nHeapArena and nDirectArena.
- // Assuming each arena has 3 chunks, the pool should not consume more than 50% of max memory.
- final Runtime runtime = Runtime.getRuntime();
- final int defaultChunkSize = DEFAULT_PAGE_SIZE << DEFAULT_MAX_ORDER;
- DEFAULT_NUM_HEAP_ARENA = Math.max(0,
- SystemPropertyUtil.getInt(
- "io.netty.allocator.numHeapArenas",
- (int) Math.min(
- runtime.availableProcessors(),
- Runtime.getRuntime().maxMemory() / defaultChunkSize / 2 / 3)));
- DEFAULT_NUM_DIRECT_ARENA = Math.max(0,
- SystemPropertyUtil.getInt(
- "io.netty.allocator.numDirectArenas",
- (int) Math.min(
- runtime.availableProcessors(),
- PlatformDependent.maxDirectMemory() / defaultChunkSize / 2 / 3)));
-
- if (logger.isDebugEnabled()) {
- logger.debug("-Dio.netty.allocator.numHeapArenas: {}", DEFAULT_NUM_HEAP_ARENA);
- logger.debug("-Dio.netty.allocator.numDirectArenas: {}", DEFAULT_NUM_DIRECT_ARENA);
- if (pageSizeFallbackCause == null) {
- logger.debug("-Dio.netty.allocator.pageSize: {}", DEFAULT_PAGE_SIZE);
- } else {
- logger.debug("-Dio.netty.allocator.pageSize: {}", DEFAULT_PAGE_SIZE, pageSizeFallbackCause);
- }
- if (maxOrderFallbackCause == null) {
- logger.debug("-Dio.netty.allocator.maxOrder: {}", DEFAULT_MAX_ORDER);
- } else {
- logger.debug("-Dio.netty.allocator.maxOrder: {}", DEFAULT_MAX_ORDER, maxOrderFallbackCause);
- }
- logger.debug("-Dio.netty.allocator.chunkSize: {}", DEFAULT_PAGE_SIZE << DEFAULT_MAX_ORDER);
- }
- }
-
- public static final PooledByteBufAllocatorL DEFAULT =
- new PooledByteBufAllocatorL(PlatformDependent.directBufferPreferred());
-
- private final PoolArenaL<byte[]>[] heapArenas;
- private final PoolArenaL<ByteBuffer>[] directArenas;
-
- final ThreadLocal<PoolThreadCacheL> threadCache = new ThreadLocal<PoolThreadCacheL>() {
- private final AtomicInteger index = new AtomicInteger();
- @Override
- protected PoolThreadCacheL initialValue() {
- final int idx = index.getAndIncrement();
- final PoolArenaL<byte[]> heapArena;
- final PoolArenaL<ByteBuffer> directArena;
-
- if (heapArenas != null) {
- heapArena = heapArenas[Math.abs(idx % heapArenas.length)];
- } else {
- heapArena = null;
- }
-
- if (directArenas != null) {
- directArena = directArenas[Math.abs(idx % directArenas.length)];
- } else {
- directArena = null;
- }
-
- return new PoolThreadCacheL(heapArena, directArena);
- }
- };
-
- public PooledByteBufAllocatorL() {
- this(false);
- }
-
- public PooledByteBufAllocatorL(boolean preferDirect) {
- this(preferDirect, DEFAULT_NUM_HEAP_ARENA, DEFAULT_NUM_DIRECT_ARENA, DEFAULT_PAGE_SIZE, DEFAULT_MAX_ORDER);
- }
-
- public PooledByteBufAllocatorL(int nHeapArena, int nDirectArena, int pageSize, int maxOrder) {
- this(false, nHeapArena, nDirectArena, pageSize, maxOrder);
- }
-
- public PooledByteBufAllocatorL(boolean preferDirect, int nHeapArena, int nDirectArena, int pageSize, int maxOrder) {
- super(preferDirect);
-
- final int chunkSize = validateAndCalculateChunkSize(pageSize, maxOrder);
-
- if (nHeapArena < 0) {
- throw new IllegalArgumentException("nHeapArena: " + nHeapArena + " (expected: >= 0)");
- }
- if (nDirectArena < 0) {
- throw new IllegalArgumentException("nDirectArea: " + nDirectArena + " (expected: >= 0)");
- }
-
- int pageShifts = validateAndCalculatePageShifts(pageSize);
-
- if (nHeapArena > 0) {
- heapArenas = newArenaArray(nHeapArena);
- for (int i = 0; i < heapArenas.length; i ++) {
- heapArenas[i] = new PoolArenaL.HeapArena(this, pageSize, maxOrder, pageShifts, chunkSize);
- }
- } else {
- heapArenas = null;
- }
-
- if (nDirectArena > 0) {
- directArenas = newArenaArray(nDirectArena);
- for (int i = 0; i < directArenas.length; i ++) {
- directArenas[i] = new PoolArenaL.DirectArena(this, pageSize, maxOrder, pageShifts, chunkSize);
- }
- } else {
- directArenas = null;
- }
- }
-
- @SuppressWarnings("unchecked")
- private static <T> PoolArenaL<T>[] newArenaArray(int size) {
- return new PoolArenaL[size];
- }
-
- private static int validateAndCalculatePageShifts(int pageSize) {
- if (pageSize < MIN_PAGE_SIZE) {
- throw new IllegalArgumentException("pageSize: " + pageSize + " (expected: 4096+)");
- }
-
- // Ensure pageSize is power of 2.
- boolean found1 = false;
- int pageShifts = 0;
- for (int i = pageSize; i != 0 ; i >>= 1) {
- if ((i & 1) != 0) {
- if (!found1) {
- found1 = true;
- } else {
- throw new IllegalArgumentException("pageSize: " + pageSize + " (expected: power of 2");
- }
- } else {
- if (!found1) {
- pageShifts ++;
- }
- }
- }
- return pageShifts;
- }
-
- private static int validateAndCalculateChunkSize(int pageSize, int maxOrder) {
- if (maxOrder > 14) {
- throw new IllegalArgumentException("maxOrder: " + maxOrder + " (expected: 0-14)");
- }
-
- // Ensure the resulting chunkSize does not overflow.
- int chunkSize = pageSize;
- for (int i = maxOrder; i > 0; i --) {
- if (chunkSize > MAX_CHUNK_SIZE / 2) {
- throw new IllegalArgumentException(String.format(
- "pageSize (%d) << maxOrder (%d) must not exceed %d", pageSize, maxOrder, MAX_CHUNK_SIZE));
- }
- chunkSize <<= 1;
- }
- return chunkSize;
- }
-
- @Override
- protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) {
- PoolThreadCacheL cache = threadCache.get();
- PoolArenaL<byte[]> heapArena = cache.heapArena;
- if (heapArena != null) {
- return heapArena.allocate(cache, initialCapacity, maxCapacity);
- } else {
- throw new UnsupportedOperationException();
- }
- }
-
- @Override
- public ByteBuf directBuffer(int initialCapacity, int maxCapacity) {
- if (initialCapacity == 0 && maxCapacity == 0) {
- return newDirectBuffer(0,0);
- }
- return super.directBuffer(initialCapacity, maxCapacity);
- }
-
- /**
- * Override the abstract allocator. Normally, the abstract allocator
- * defaults the second parameter to MAXINT as a "sanity check", but we
- * have reinterpreted the second parameter as "max requested".
- */
- @Override
- public ByteBuf directBuffer(int request) {
- return directBuffer(request, request);
- }
-
-
-
- /**
- * Allocate a buffer from the current thread's direct arena.
- */
- @Override
- protected ByteBuf newDirectBuffer(int minRequested, int maxRequested) {
- PoolThreadCacheL cache = threadCache.get();
- PoolArenaL<ByteBuffer> directArena = cache.directArena;
- if (directArena != null) {
- return directArena.allocate(cache, minRequested, maxRequested);
- } else {
- if (PlatformDependent.hasUnsafe()) {
- throw new UnsupportedOperationException();
-// return new UnpooledUnsafeDirectByteBuf(this, initialCapacity, maxCapacity);
- } else {
- throw new UnsupportedOperationException();
-// return new UnpooledDirectByteBuf(this, initialCapacity, maxCapacity);
- }
- }
- }
-
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append(heapArenas.length);
- buf.append(" heap arena(s):");
- buf.append(StringUtil.NEWLINE);
- for (PoolArenaL<byte[]> a: heapArenas) {
- buf.append(a);
- }
- buf.append(directArenas.length);
- buf.append(" direct arena(s):");
- buf.append(StringUtil.NEWLINE);
- for (PoolArenaL<ByteBuffer> a: directArenas) {
- buf.append(a);
- }
- return buf.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/af29c255/exec/bufferl/src/main/java/io/netty/buffer/PooledByteBufL.java
----------------------------------------------------------------------
diff --git a/exec/bufferl/src/main/java/io/netty/buffer/PooledByteBufL.java b/exec/bufferl/src/main/java/io/netty/buffer/PooledByteBufL.java
deleted file mode 100644
index 3f871e5..0000000
--- a/exec/bufferl/src/main/java/io/netty/buffer/PooledByteBufL.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import io.netty.buffer.AbstractReferenceCountedByteBuf;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.util.Recycler;
-import io.netty.util.ResourceLeak;
-import io.netty.util.ResourceLeakDetector;
-
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-
-/**
- * A ByteBuf optimized to work with little endian direct buffers
- * and the netty pool allocator.
- * The buffer can be of any size - tiny, small, normal, or huge.
- * The class contains all information needed to free the memory
- * (which chunk, which pages, which elements).
- *
- * @param <T>
- */
-
-abstract class PooledByteBufL<T> extends AbstractReferenceCountedByteBuf {
-
- private final ResourceLeak leak;
- private final Recycler.Handle recyclerHandle;
-
- protected PoolChunkL<T> chunk;
- protected long handle;
- protected T memory;
- protected int offset;
- protected int length;
- private int maxLength;
-
- private ByteBuffer tmpNioBuf;
-
- protected PooledByteBufL(Recycler.Handle recyclerHandle, int maxCapacity) {
- super(maxCapacity);
- leak = leakDetector.open(this);
- this.recyclerHandle = recyclerHandle;
- }
-
- /**
- * Initialize a new buffer for "normal" allocations.
- * @param chunk - which chunk the buffer came from
- * @param handle - which pages within the chunk
- * @param offset - byte offset to the first page
- * @param length - the requested length
- * @param maxLength - the max limit for resizing.
- */
- void init(PoolChunkL<T> chunk, long handle, int offset, int length, int maxLength) {
- assert handle >= 0;
- assert chunk != null;
-
- this.chunk = chunk;
- this.handle = handle;
- memory = chunk.memory;
- this.offset = offset;
- this.length = length;
- this.maxLength = maxLength;
- setIndex(0, 0);
- tmpNioBuf = null;
- }
-
- void initUnpooled(PoolChunkL<T> chunk, int length) {
- assert chunk != null;
-
- this.chunk = chunk;
- handle = 0;
- memory = chunk.memory;
- offset = 0;
- this.length = maxLength = length;
- setIndex(0, 0);
- tmpNioBuf = null;
- }
-
- @Override
- public final int capacity() {
- return length;
- }
-
-
- /**
- * Change the size of an allocated buffer, reallocating if appropriate.
- * @param newCapacity
- * @return
- */
- @Override
- public final ByteBuf capacity(int newCapacity) {
- ensureAccessible();
-
- if (chunk.parent == null) {
- return this; //TODO figure out if this is the correct behavior
- }
-
- // Check for the easy resizing cases, and return if successfully resized.
- if (chunk.unpooled) {
- if (newCapacity == length) {
- return this;
- }
- } else {
- if (newCapacity > length) {
- if (newCapacity <= maxLength) {
- length = newCapacity;
- return this;
- }
- } else if (newCapacity < length) {
- if (newCapacity > maxLength >>> 1) {
- if (maxLength <= 512) {
- if (newCapacity > maxLength - 16) {
- length = newCapacity;
- setIndex(Math.min(readerIndex(), newCapacity), Math.min(writerIndex(), newCapacity));
- return this;
- }
- } else { // > 512 (i.e. >= 1024)
- length = newCapacity;
- setIndex(Math.min(readerIndex(), newCapacity), Math.min(writerIndex(), newCapacity));
- return this;
- }
- }
- } else {
- return this;
- }
- }
-
- // Trim down the size of the current buffer, if able to.
- long newHandle = chunk.parent.trim(chunk, handle, newCapacity);
- if (newHandle != -1) {
- chunk.initBuf(this, newHandle, newCapacity);
- return this;
- }
-
- // Reallocate the data.
- chunk.arena.reallocate(this, newCapacity, true);
-
- return this;
- }
-
-
-
- @Override
- public final ByteBufAllocator alloc() {
- return chunk.arena.parent;
- }
-
- @Override
- public final ByteOrder order() {
- return ByteOrder.BIG_ENDIAN; // TODO: Is this correct?
- }
-
- @Override
- public final ByteBuf unwrap() {
- return null;
- }
-
- protected final ByteBuffer internalNioBuffer() {
- ByteBuffer tmpNioBuf = this.tmpNioBuf;
- if (tmpNioBuf == null) {
- this.tmpNioBuf = tmpNioBuf = newInternalNioBuffer(memory);
- }
- return tmpNioBuf;
- }
-
- protected abstract ByteBuffer newInternalNioBuffer(T memory);
-
-
- /**
- * Free the memory and recycle the header.
- */
- @Override
- protected final void deallocate() {
- if (handle >= 0) {
- final long handle = this.handle;
- this.handle = -1;
- memory = null;
- chunk.arena.free(chunk, handle);
- if (leak != null) {
- leak.close();
- } else {
- recycle();
- }
- }
- }
-
-
-
- @SuppressWarnings("unchecked")
- private void recycle() {
- Recycler.Handle recyclerHandle = this.recyclerHandle;
- if (recyclerHandle != null) {
- ((Recycler<Object>) recycler()).recycle(this, recyclerHandle);
- }
- }
-
- protected abstract Recycler<?> recycler();
-
- protected final int idx(int index) {
- return offset + index;
- }
-}
-
-
-/**
- * Exception thrown after resizing a buffer results in excessive copying.
- * The buffer has been properly resized, so It is possible to ignore the exception and continue.
- */
-class TooMuchCopyingException extends IllegalArgumentException {
- private static final long serialVersionUID = -8184712014469036532L;
- TooMuchCopyingException(String s) {
- super(s);
- }
-}
[06/14] git commit: DRILL-1068: Reduce memory consumption of various
operators. Add more memory stats to query profile.
Posted by ja...@apache.org.
DRILL-1068: Reduce memory consumption of various operators. Add more memory stats to query profile.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c3eea13c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c3eea13c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c3eea13c
Branch: refs/heads/master
Commit: c3eea13ce4c45e6bcd0b6863571e1561a9fa1931
Parents: 0dfeac8
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Jun 21 13:01:43 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:09:39 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/ops/FragmentStats.java | 6 +-
.../apache/drill/exec/ops/OperatorContext.java | 2 +-
.../apache/drill/exec/ops/OperatorStats.java | 16 +++-
.../org/apache/drill/exec/ops/SenderStats.java | 11 +--
.../drill/exec/physical/impl/BaseRootExec.java | 4 +-
.../exec/physical/impl/TopN/TopNBatch.java | 6 +-
.../impl/aggregate/HashAggTemplate.java | 66 ++++++++--------
.../impl/aggregate/StreamingAggBatch.java | 5 +-
.../impl/aggregate/StreamingAggTemplate.java | 14 ++--
.../impl/aggregate/StreamingAggregator.java | 5 +-
.../physical/impl/filter/FilterRecordBatch.java | 7 +-
.../exec/physical/impl/join/HashJoinBatch.java | 17 ++--
.../impl/join/HashJoinProbeTemplate.java | 82 ++++++++++++--------
.../impl/mergereceiver/MergingRecordBatch.java | 28 +++----
.../PartitionSenderRootExec.java | 4 +-
.../partitionsender/PartitionerTemplate.java | 7 +-
.../exec/physical/impl/svremover/Copier.java | 2 +-
.../impl/svremover/CopierTemplate2.java | 9 +--
.../impl/svremover/CopierTemplate4.java | 7 +-
.../impl/svremover/RemovingRecordBatch.java | 17 +---
.../planner/fragment/SimpleParallelizer.java | 3 +-
.../drill/exec/record/VectorContainer.java | 21 ++++-
.../exec/vector/allocator/VectorAllocator.java | 29 +++----
23 files changed, 184 insertions(+), 184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentStats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentStats.java
index 19ac0aa..22872f9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentStats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentStats.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.ops;
import java.util.List;
-import org.apache.drill.exec.proto.BitControl.FragmentStatus;
+import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile;
import com.codahale.metrics.MetricRegistry;
@@ -46,8 +46,8 @@ public class FragmentStats {
}
}
- public OperatorStats getOperatorStats(OpProfileDef profileDef){
- OperatorStats stats = new OperatorStats(profileDef);
+ public OperatorStats getOperatorStats(OpProfileDef profileDef, BufferAllocator allocator){
+ OperatorStats stats = new OperatorStats(profileDef, allocator);
if(profileDef.operatorType != -1){
operators.add(stats);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContext.java
index d62ea2f..2d46733 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContext.java
@@ -37,7 +37,7 @@ public class OperatorContext implements Closeable {
this.popConfig = popConfig;
OpProfileDef def = new OpProfileDef(popConfig.getOperatorId(), popConfig.getOperatorType(), getChildCount(popConfig));
- this.stats = context.getStats().getOperatorStats(def);
+ this.stats = context.getStats().getOperatorStats(def, allocator);
}
public OperatorContext(PhysicalOperator popConfig, FragmentContext context, OperatorStats stats) throws OutOfMemoryException {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
index dcb73c8..bd8d899 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.ops;
-import org.apache.commons.collections.Buffer;
+import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared.MetricValue;
import org.apache.drill.exec.proto.UserBitShared.OperatorProfile;
import org.apache.drill.exec.proto.UserBitShared.StreamProfile;
@@ -30,6 +30,7 @@ public class OperatorStats {
protected final int operatorId;
protected final int operatorType;
+ private final BufferAllocator allocator;
private IntLongOpenHashMap longMetrics = new IntLongOpenHashMap();
private IntDoubleOpenHashMap doubleMetrics = new IntDoubleOpenHashMap();
@@ -53,12 +54,13 @@ public class OperatorStats {
private long schemas;
- public OperatorStats(OpProfileDef def){
- this(def.getOperatorId(), def.getOperatorType(), def.getIncomingCount());
+ public OperatorStats(OpProfileDef def, BufferAllocator allocator){
+ this(def.getOperatorId(), def.getOperatorType(), def.getIncomingCount(), allocator);
}
- private OperatorStats(int operatorId, int operatorType, int inputCount) {
+ private OperatorStats(int operatorId, int operatorType, int inputCount, BufferAllocator allocator) {
super();
+ this.allocator = allocator;
this.operatorId = operatorId;
this.operatorType = operatorType;
this.recordsReceivedByInput = new long[inputCount];
@@ -126,6 +128,12 @@ public class OperatorStats {
.setProcessNanos(processingNanos)
.setWaitNanos(waitNanos);
+ if(allocator != null){
+ b.setLocalMemoryAllocated(allocator.getAllocatedMemory());
+ }
+
+
+
addAllMetrics(b);
return b.build();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/ops/SenderStats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/SenderStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/SenderStats.java
index c766632..5167edb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/SenderStats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/SenderStats.java
@@ -17,21 +17,22 @@
*/
package org.apache.drill.exec.ops;
+import java.util.List;
+
+import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.impl.partitionsender.PartitionStatsBatch;
import org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderStats;
+import org.apache.drill.exec.physical.impl.partitionsender.PartitionStatsBatch;
import org.apache.drill.exec.proto.UserBitShared;
-import java.util.List;
-
public class SenderStats extends OperatorStats {
long minReceiverRecordCount = 0;
long maxReceiverRecordCount = 0;
int nSenders = 0;
- public SenderStats(PhysicalOperator operator) {
- super(new OpProfileDef(operator.getOperatorId(), operator.getOperatorType(), OperatorContext.getChildCount(operator)));
+ public SenderStats(PhysicalOperator operator, BufferAllocator allocator) {
+ super(new OpProfileDef(operator.getOperatorId(), operator.getOperatorType(), OperatorContext.getChildCount(operator)), allocator);
}
public void updatePartitionStats(List<? extends PartitionStatsBatch> outgoing) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
index 452052b..d56da51 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
@@ -32,9 +32,9 @@ public abstract class BaseRootExec implements RootExec {
protected OperatorContext oContext = null;
public BaseRootExec(FragmentContext context, PhysicalOperator config) throws OutOfMemoryException {
- this.stats = new SenderStats(config);
- context.getStats().addOperatorStats(this.stats);
this.oContext = new OperatorContext(config, context, stats);
+ this.stats = new SenderStats(config, oContext.getAllocator());
+ context.getStats().addOperatorStats(this.stats);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
index 4a5d368..846d419 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
@@ -219,14 +219,12 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
if (copier == null) {
copier = RemovingRecordBatch.getGenerated4Copier(batch, context, oContext.getAllocator(), newContainer, newBatch);
} else {
- List<VectorAllocator> allocators = Lists.newArrayList();
for(VectorWrapper<?> i : batch){
ValueVector v = TypeHelper.getNewVector(i.getField(), oContext.getAllocator());
newContainer.add(v);
- allocators.add(RemovingRecordBatch.getAllocator4(v));
}
- copier.setupRemover(context, batch, newBatch, allocators.toArray(new VectorAllocator[allocators.size()]));
+ copier.setupRemover(context, batch, newBatch);
}
SortRecordBatchBuilder builder = new SortRecordBatchBuilder(oContext.getAllocator(), MAX_SORT_BYTES);
do {
@@ -372,7 +370,7 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
public Iterator<VectorWrapper<?>> iterator() {
return container.iterator();
}
-
+
@Override
public VectorContainer getOutgoingContainer() {
throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
index 935bbb3..1afa5ae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
@@ -148,7 +148,7 @@ public abstract class HashAggTemplate implements HashAggregator {
private void clear() {
aggrValuesContainer.clear();
}
-
+
private int getNumGroups() {
return maxOccupiedIdx + 1;
}
@@ -156,7 +156,7 @@ public abstract class HashAggTemplate implements HashAggregator {
private int getOutputCount() {
return batchOutputCount;
}
-
+
// Code-generated methods (implemented in HashAggBatch)
@RuntimeOverridden
@@ -242,7 +242,8 @@ public abstract class HashAggTemplate implements HashAggregator {
if (EXTRA_DEBUG_1) logger.debug ("Starting outer loop of doWork()...");
for (; underlyingIndex < incoming.getRecordCount(); incIndex()) {
if(EXTRA_DEBUG_2) logger.debug("Doing loop with values underlying {}, current {}", underlyingIndex, currentIndex);
- checkGroupAndAggrValues(currentIndex);
+ boolean success = checkGroupAndAggrValues(currentIndex);
+ assert success : "HashAgg couldn't copy values.";
}
if (EXTRA_DEBUG_1) logger.debug("Processed {} records", underlyingIndex);
@@ -273,7 +274,8 @@ public abstract class HashAggTemplate implements HashAggregator {
if(incoming.getRecordCount() == 0){
continue;
} else {
- checkGroupAndAggrValues(currentIndex);
+ boolean success = checkGroupAndAggrValues(currentIndex);
+ assert success : "HashAgg couldn't copy values.";
incIndex();
if(EXTRA_DEBUG_1) logger.debug("Continuing outside loop");
@@ -282,16 +284,16 @@ public abstract class HashAggTemplate implements HashAggregator {
case NONE:
// outcome = out;
-
+
buildComplete = true;
-
+
updateStats(htable);
- // output the first batch; remaining batches will be output
+ // output the first batch; remaining batches will be output
// in response to each next() call by a downstream operator
-
+
outputCurrentBatch();
-
+
// cleanup incoming batch since output of aggregation does not need
// any references to the incoming
@@ -314,7 +316,7 @@ public abstract class HashAggTemplate implements HashAggregator {
if(first) first = !first;
}
}
-
+
private void allocateOutgoing(int numOutputRecords) {
for (VectorAllocator a : keyAllocators) {
@@ -326,7 +328,7 @@ public abstract class HashAggTemplate implements HashAggregator {
if(EXTRA_DEBUG_2) logger.debug("Outgoing batch: Allocating {} with {} records.", a, numOutputRecords);
a.alloc(numOutputRecords);
}
-
+
}
@Override
@@ -398,12 +400,12 @@ public abstract class HashAggTemplate implements HashAggregator {
bh.setup();
}
-
+
// output the keys and values for a particular batch holder
private boolean outputKeysAndValues(int batchIdx) {
-
+
allocateOutgoing(batchIdx);
-
+
if (! this.htable.outputKeys(batchIdx)) {
return false;
}
@@ -412,14 +414,14 @@ public abstract class HashAggTemplate implements HashAggregator {
}
outBatchIndex = batchIdx+1;
-
+
if (outBatchIndex == batchHolders.size()) {
allFlushed = true;
}
-
+
return true;
}
-
+
public IterOutcome outputCurrentBatch() {
if (outBatchIndex >= batchHolders.size()) {
this.outcome = IterOutcome.NONE;
@@ -433,35 +435,35 @@ public abstract class HashAggTemplate implements HashAggregator {
this.outcome = IterOutcome.NONE;
return outcome;
}
-
+
allocateOutgoing(batchOutputRecords);
-
+
boolean outputKeysStatus = this.htable.outputKeys(outBatchIndex) ;
- boolean outputValuesStatus = batchHolders.get(outBatchIndex).outputValues();
+ boolean outputValuesStatus = batchHolders.get(outBatchIndex).outputValues();
if (outputKeysStatus && outputValuesStatus) {
-
+
// set the value count for outgoing batch value vectors
for(VectorWrapper<?> v : outgoing) {
v.getValueVector().getMutator().setValueCount(batchOutputRecords);
}
-
+
outputCount += batchOutputRecords;
-
+
if(first){
this.outcome = IterOutcome.OK_NEW_SCHEMA;
}else{
this.outcome = IterOutcome.OK;
}
-
+
logger.debug("HashAggregate: Output current batch index {} with {} records.", outBatchIndex, batchOutputRecords);
lastBatchOutputCount = batchOutputRecords;
outBatchIndex++;
if (outBatchIndex == batchHolders.size()) {
allFlushed = true;
-
+
logger.debug("HashAggregate: All batches flushed.");
-
+
// cleanup my internal state since there is nothing more to return
this.cleanup();
}
@@ -470,18 +472,18 @@ public abstract class HashAggTemplate implements HashAggregator {
if (!outputValuesStatus) context.fail(new Exception("Failed to output values for current batch !"));
this.outcome = IterOutcome.STOP;
}
-
+
return this.outcome;
}
-
+
public boolean allFlushed() {
return allFlushed;
}
-
+
public boolean buildComplete() {
return buildComplete;
}
-
+
public int numGroupedRecords() {
return numGroupedRecords;
}
@@ -545,12 +547,12 @@ public abstract class HashAggTemplate implements HashAggregator {
return false;
}
-
+
private void updateStats(HashTable htable) {
htable.getStats(htStats);
this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_BUCKETS, htStats.numBuckets);
this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_ENTRIES, htStats.numEntries);
- this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_RESIZING, htStats.numResizing);
+ this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_RESIZING, htStats.numResizing);
}
// Code-generated methods (implemented in HashAggBatch)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
index 367d2c7..b587ad1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
@@ -168,7 +168,6 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
private StreamingAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException, IOException{
ClassGenerator<StreamingAggregator> cg = CodeGenerator.getRoot(StreamingAggTemplate.TEMPLATE_DEFINITION, context.getFunctionRegistry());
container.clear();
- List<VectorAllocator> allocators = Lists.newArrayList();
LogicalExpression[] keyExprs = new LogicalExpression[popConfig.getKeys().length];
LogicalExpression[] valueExprs = new LogicalExpression[popConfig.getExprs().length];
@@ -183,7 +182,6 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
keyExprs[i] = expr;
final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
- allocators.add(VectorAllocator.getAllocator(vector, 50));
keyOutputIds[i] = container.add(vector);
}
@@ -194,7 +192,6 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
- allocators.add(VectorAllocator.getAllocator(vector, 50));
TypedFieldId id = container.add(vector);
valueExprs[i] = new ValueVectorWriteExpression(id, expr, true);
}
@@ -212,7 +209,7 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
container.buildSchema(SelectionVectorMode.NONE);
StreamingAggregator agg = context.getImplementationClass(cg);
- agg.setup(context, incoming, this, allocators.toArray(new VectorAllocator[allocators.size()]));
+ agg.setup(context, incoming, this);
return agg;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
index 3bd861d..4d6e7c4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
@@ -25,6 +25,7 @@ import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatch.IterOutcome;
import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
public abstract class StreamingAggTemplate implements StreamingAggregator {
@@ -44,18 +45,15 @@ public abstract class StreamingAggTemplate implements StreamingAggregator {
private RecordBatch incoming;
private BatchSchema schema;
private StreamingAggBatch outgoing;
- private VectorAllocator[] allocators;
private FragmentContext context;
private InternalBatch remainderBatch;
@Override
- public void setup(FragmentContext context, RecordBatch incoming, StreamingAggBatch outgoing, VectorAllocator[] allocators) throws SchemaChangeException {
- this.allocators = allocators;
+ public void setup(FragmentContext context, RecordBatch incoming, StreamingAggBatch outgoing) throws SchemaChangeException {
this.context = context;
this.incoming = incoming;
this.schema = incoming.getSchema();
- this.allocators = allocators;
this.outgoing = outgoing;
setupInterior(incoming, outgoing);
this.currentIndex = incoming.getRecordCount() == 0 ? 0 : this.getVectorIndex(underlyingIndex);
@@ -63,10 +61,8 @@ public abstract class StreamingAggTemplate implements StreamingAggregator {
private void allocateOutgoing() {
- for (VectorAllocator a : allocators) {
- if(EXTRA_DEBUG) logger.debug("Allocating {} with {} records.", a, 20000);
- a.alloc(20000);
- if(EXTRA_DEBUG) logger.debug("Allocated {}", a);
+ for(VectorWrapper<?> w : outgoing){
+ w.getValueVector().allocateNew();
}
}
@@ -90,7 +86,7 @@ public abstract class StreamingAggTemplate implements StreamingAggregator {
public AggOutcome doWork() {
try{ // outside loop to ensure that first is set to false after the first run.
outputCount = 0;
-
+
// if we're in the first state, allocate outgoing.
if(first){
allocateOutgoing();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
index 52f30ae..c624c9a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
@@ -31,9 +31,8 @@ public interface StreamingAggregator {
public static enum AggOutcome {
RETURN_OUTCOME, CLEANUP_AND_RETURN, UPDATE_AGGREGATOR;
}
-
- public abstract void setup(FragmentContext context, RecordBatch incoming, StreamingAggBatch outgoing,
- VectorAllocator[] allocators) throws SchemaChangeException;
+
+ public abstract void setup(FragmentContext context, RecordBatch incoming, StreamingAggBatch outgoing) throws SchemaChangeException;
public abstract IterOutcome getOutcome();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
index d52d2e3..89a6d09 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
@@ -55,7 +55,7 @@ public class FilterRecordBatch extends AbstractSingleRecordBatch<Filter>{
public FilterRecordBatch(Filter pop, RecordBatch incoming, FragmentContext context) throws OutOfMemoryException {
super(pop, context, incoming);
}
-
+
@Override
public FragmentContext getContext() {
return context;
@@ -85,8 +85,8 @@ public class FilterRecordBatch extends AbstractSingleRecordBatch<Filter>{
// m.setValueCount(recordCount);
// }
}
-
-
+
+
@Override
public void cleanup() {
if(sv2 != null) sv2.clear();
@@ -128,7 +128,6 @@ public class FilterRecordBatch extends AbstractSingleRecordBatch<Filter>{
protected Filterer generateSV4Filterer() throws SchemaChangeException {
final ErrorCollector collector = new ErrorCollectorImpl();
final List<TransferPair> transfers = Lists.newArrayList();
- final List<VectorAllocator> allocators = Lists.newArrayList();
final ClassGenerator<Filterer> cg = CodeGenerator.getRoot(Filterer.TEMPLATE_DEFINITION4, context.getFunctionRegistry());
final LogicalExpression expr = ExpressionTreeMaterializer.materialize(popConfig.getExpr(), incoming, collector, context.getFunctionRegistry());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 1c028d0..bd0e23f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -103,9 +103,6 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
// Current batch index on the build side
private int buildBatchIndex = 0;
- // List of vector allocators
- private List<VectorAllocator> allocators = null;
-
// Schema of the build side
private BatchSchema rightSchema = null;
@@ -261,8 +258,8 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
}
}
- HashTableConfig htConfig =
- new HashTableConfig(context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE_KEY).num_val.intValue(),
+ HashTableConfig htConfig =
+ new HashTableConfig(context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE_KEY).num_val.intValue(),
HashTable.DEFAULT_LOAD_FACTOR, rightExpr, leftExpr);
// Create the chained hash table
@@ -346,7 +343,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
public HashJoinProbe setupHashJoinProbe() throws ClassTransformationException, IOException {
- allocators = new ArrayList<>();
+
final CodeGenerator<HashJoinProbe> cg = CodeGenerator.get(HashJoinProbe.TEMPLATE_DEFINITION, context.getFunctionRegistry());
ClassGenerator<HashJoinProbe> g = cg.getRoot();
@@ -374,7 +371,6 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
// Add the vector to our output container
ValueVector v = TypeHelper.getNewVector(MaterializedField.create(vv.getField().getPath(), outputType), context.getAllocator());
container.add(v);
- allocators.add(RemovingRecordBatch.getAllocator4(v));
JVar inVV = g.declareVectorValueSetupAndMember("buildBatch", new TypedFieldId(vv.getField().getType(), true, fieldId));
JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, fieldId));
@@ -410,7 +406,6 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
ValueVector v = TypeHelper.getNewVector(MaterializedField.create(vv.getField().getPath(), outputType), oContext.getAllocator());
container.add(v);
- allocators.add(RemovingRecordBatch.getAllocator4(v));
JVar inVV = g.declareVectorValueSetupAndMember("probeBatch", new TypedFieldId(inputType, false, fieldId));
JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, outputFieldId));
@@ -432,9 +427,9 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
}
private void allocateVectors(){
- for(VectorAllocator a : allocators){
- a.alloc(RecordBatch.MAX_BATCH_SIZE);
- }
+ for(VectorWrapper<?> v : container){
+ v.getValueVector().allocateNew();
+ }
}
public HashJoinBatch(HashJoinPOP popConfig, FragmentContext context, RecordBatch left, RecordBatch right) throws OutOfMemoryException {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
index a4cc662..21c4ae7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
@@ -41,9 +41,11 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe {
// Join type, INNER, LEFT, RIGHT or OUTER
private JoinRelType joinType;
-
+
private HashJoinBatch outgoingJoinBatch = null;
+ private static final int TARGET_RECORDS_PER_BATCH = 4000;
+
/* Helper class
* Maintains linked list of build side records with the same key
* Keeps information about which build records have a corresponding
@@ -91,14 +93,21 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe {
}
public void executeProjectRightPhase() {
- while (outputRecords < RecordBatch.MAX_BATCH_SIZE && recordsProcessed < recordsToProcess) {
- boolean success = projectBuildRecord(unmatchedBuildIndexes.get(recordsProcessed++), outputRecords++);
- assert success;
+ boolean success = true;
+ while (outputRecords < TARGET_RECORDS_PER_BATCH && recordsProcessed < recordsToProcess) {
+ success = projectBuildRecord(unmatchedBuildIndexes.get(recordsProcessed), outputRecords);
+ if(success){
+ recordsProcessed++;
+ outputRecords++;
+ }else{
+ if(outputRecords == 0) throw new IllegalStateException("Too big to fail.");
+ break;
+ }
}
}
public void executeProbePhase() throws SchemaChangeException {
- while (outputRecords < RecordBatch.MAX_BATCH_SIZE && recordsToProcess > 0) {
+ while (outputRecords < TARGET_RECORDS_PER_BATCH && recordsToProcess > 0) {
// Check if we have processed all records in this batch we need to invoke next
if (recordsProcessed == recordsToProcess) {
@@ -157,31 +166,34 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe {
*/
hjHelper.setRecordMatched(currentCompositeIdx);
- boolean success = projectBuildRecord(currentCompositeIdx, outputRecords);
- assert success;
- success = projectProbeRecord(recordsProcessed, outputRecords);
- assert success;
- outputRecords++;
+ boolean success = projectBuildRecord(currentCompositeIdx, outputRecords) //
+ && projectProbeRecord(recordsProcessed, outputRecords);
+ if(!success){
+ // we failed to project. redo this record.
+ getNextRecord = false;
+ }else{
+ outputRecords++;
- /* Projected single row from the build side with matching key but there
- * may be more rows with the same key. Check if that's the case
- */
- currentCompositeIdx = hjHelper.getNextIndex(currentCompositeIdx);
- if (currentCompositeIdx == -1) {
- /* We only had one row in the build side that matched the current key
- * from the probe side. Drain the next row in the probe side.
+ /* Projected single row from the build side with matching key but there
+ * may be more rows with the same key. Check if that's the case
*/
- recordsProcessed++;
+ currentCompositeIdx = hjHelper.getNextIndex(currentCompositeIdx);
+ if (currentCompositeIdx == -1) {
+ /* We only had one row in the build side that matched the current key
+ * from the probe side. Drain the next row in the probe side.
+ */
+ recordsProcessed++;
+ }
+ else {
+ /* There is more than one row with the same key on the build side
+ * don't drain more records from the probe side till we have projected
+ * all the rows with this key
+ */
+ getNextRecord = false;
+ }
}
- else {
- /* There is more than one row with the same key on the build side
- * don't drain more records from the probe side till we have projected
- * all the rows with this key
- */
- getNextRecord = false;
- }
- }
- else { // No matching key
+
+ } else { // No matching key
// If we have a left outer join, project the keys
if (joinType == JoinRelType.LEFT || joinType == JoinRelType.FULL) {
@@ -190,12 +202,18 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe {
}
recordsProcessed++;
}
- }
- else {
+ } else {
hjHelper.setRecordMatched(currentCompositeIdx);
- boolean success = projectBuildRecord(currentCompositeIdx, outputRecords);
- assert success;
- success = projectProbeRecord(recordsProcessed, outputRecords);
+ boolean success = projectBuildRecord(currentCompositeIdx, outputRecords) //
+ && projectProbeRecord(recordsProcessed, outputRecords);
+ if(!success){
+ if(outputRecords == 0){
+ throw new IllegalStateException("Record larger than single batch.");
+ }else{
+ // we've output some records but failed to output this one. return and wait for next call.
+ return;
+ }
+ }
assert success;
outputRecords++;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
index 9351844..a16d64c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
@@ -90,14 +90,12 @@ public class MergingRecordBatch extends AbstractRecordBatch<MergingReceiverPOP>
private boolean hasRun = false;
private boolean prevBatchWasFull = false;
private boolean hasMoreIncoming = true;
- private final int DEFAULT_ALLOC_RECORD_COUNT = 20000;
private int outgoingPosition = 0;
private int senderCount = 0;
private RawFragmentBatch[] incomingBatches;
private int[] batchOffsets;
private PriorityQueue <Node> pqueue;
- private List<VectorAllocator> allocators;
private RawFragmentBatch emptyBatch = null;
private boolean done = false;
@@ -117,7 +115,6 @@ public class MergingRecordBatch extends AbstractRecordBatch<MergingReceiverPOP>
super(config, context);
this.fragProviders = fragProviders;
this.context = context;
- this.allocators = Lists.newArrayList();
this.outgoingContainer = new VectorContainer();
}
@@ -243,14 +240,11 @@ public class MergingRecordBatch extends AbstractRecordBatch<MergingReceiverPOP>
// allocate a new value vector
ValueVector outgoingVector = TypeHelper.getNewVector(v.getField(), oContext.getAllocator());
- VectorAllocator allocator = VectorAllocator.getAllocator(outgoingVector, 50);
- allocator.alloc(DEFAULT_ALLOC_RECORD_COUNT);
- allocators.add(allocator);
+ outgoingVector.allocateNew();
outgoingContainer.add(outgoingVector);
++vectorCount;
}
- logger.debug("Allocating {} outgoing vectors with {} values", vectorCount, DEFAULT_ALLOC_RECORD_COUNT);
schema = bldr.build();
if (schema != null && !schema.equals(schema)) {
@@ -295,11 +289,11 @@ public class MergingRecordBatch extends AbstractRecordBatch<MergingReceiverPOP>
}
pqueue.poll();
- if (isOutgoingFull()) {
- // set a flag so that we reallocate on the next iteration
- logger.debug("Outgoing vectors record batch size reached; breaking");
- prevBatchWasFull = true;
- }
+// if (isOutgoingFull()) {
+// // set a flag so that we reallocate on the next iteration
+// logger.debug("Outgoing vectors record batch size reached; breaking");
+// prevBatchWasFull = true;
+// }
if (node.valueIndex == batchLoaders[node.batchId].getRecordCount() - 1) {
// reached the end of an incoming record batch
@@ -443,14 +437,12 @@ public class MergingRecordBatch extends AbstractRecordBatch<MergingReceiverPOP>
}
private void allocateOutgoing() {
- for (VectorAllocator allocator : allocators) {
- allocator.alloc(DEFAULT_ALLOC_RECORD_COUNT);
- }
+ outgoingContainer.allocateNew();
}
- private boolean isOutgoingFull() {
- return outgoingPosition == DEFAULT_ALLOC_RECORD_COUNT;
- }
+// private boolean isOutgoingFull() {
+// return outgoingPosition == DEFAULT_ALLOC_RECORD_COUNT;
+// }
/**
* Creates a generate class which implements the copy and compare methods.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
index c4844d5..5fcfdcc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
@@ -139,7 +139,7 @@ public class PartitionSenderRootExec extends BaseRootExec {
return false;
}
stats.updatePartitionStats(partitioner.getOutgoingBatches());
- for (VectorWrapper v : incoming) {
+ for (VectorWrapper<?> v : incoming) {
v.clear();
}
return true;
@@ -156,8 +156,6 @@ public class PartitionSenderRootExec extends BaseRootExec {
final ErrorCollector collector = new ErrorCollectorImpl();
final ClassGenerator<Partitioner> cg ;
- boolean hyper = false;
-
cg = CodeGenerator.getRoot(Partitioner.TEMPLATE_DEFINITION, context.getFunctionRegistry());
ClassGenerator<Partitioner> cgInner = cg.getInnerGenerator("OutgoingRecordBatch");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
index 6a26d30..0d967b5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
@@ -212,8 +212,7 @@ public abstract class PartitionerTemplate implements Partitioner {
private int recordCount;
private int totalRecords;
private OperatorStats stats;
- private static final int DEFAULT_RECORD_BATCH_SIZE = 20000;
- private static final int DEFAULT_VARIABLE_WIDTH_SIZE = 200;
+ private static final int DEFAULT_RECORD_BATCH_SIZE = 1000;
private final StatusHandler statusHandler;
@@ -301,7 +300,7 @@ public abstract class PartitionerTemplate implements Partitioner {
recordCount = 0;
vectorContainer.zeroVectors();
for (VectorWrapper<?> v : vectorContainer) {
- VectorAllocator.getAllocator(v.getValueVector(), DEFAULT_VARIABLE_WIDTH_SIZE).alloc(DEFAULT_RECORD_BATCH_SIZE);
+ v.getValueVector().allocateNew();
}
if (!statusHandler.isOk()) {
throw new IOException(statusHandler.getException());
@@ -320,7 +319,7 @@ public abstract class PartitionerTemplate implements Partitioner {
// allocate a new value vector
ValueVector outgoingVector = TypeHelper.getNewVector(v.getField(), allocator);
- VectorAllocator.getAllocator(outgoingVector, DEFAULT_VARIABLE_WIDTH_SIZE).alloc(DEFAULT_RECORD_BATCH_SIZE);
+ outgoingVector.allocateNew();
vectorContainer.add(outgoingVector);
}
outSchema = bldr.build();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java
index 0aab7b2..dfc37c6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java
@@ -27,7 +27,7 @@ public interface Copier {
public static TemplateClassDefinition<Copier> TEMPLATE_DEFINITION2 = new TemplateClassDefinition<Copier>(Copier.class, CopierTemplate2.class);
public static TemplateClassDefinition<Copier> TEMPLATE_DEFINITION4 = new TemplateClassDefinition<Copier>(Copier.class, CopierTemplate4.class);
- public void setupRemover(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, VectorAllocator[] allocators) throws SchemaChangeException;
+ public void setupRemover(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException;
public abstract int copyRecords(int index, int recordCount);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java
index 387497c..5cc308a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java
@@ -20,26 +20,21 @@ package org.apache.drill.exec.physical.impl.svremover;
import javax.inject.Named;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.selection.SelectionVector2;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.allocator.VectorAllocator;
public abstract class CopierTemplate2 implements Copier{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CopierTemplate2.class);
private SelectionVector2 sv2;
- private RecordBatch incoming;
private RecordBatch outgoing;
@Override
- public void setupRemover(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, VectorAllocator[] allocators) throws SchemaChangeException{
+ public void setupRemover(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException{
this.sv2 = incoming.getSelectionVector2();
- this.incoming = incoming;
this.outgoing = outgoing;
doSetup(context, incoming, outgoing);
}
@@ -47,7 +42,7 @@ public abstract class CopierTemplate2 implements Copier{
@Override
public int copyRecords(int index, int recordCount){
for(VectorWrapper<?> out : outgoing){
- out.getValueVector().allocateNewSafe();
+ out.getValueVector().allocateNew();
}
int outgoingPosition = 0;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate4.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate4.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate4.java
index b48a8fd..7a1c029 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate4.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate4.java
@@ -31,13 +31,11 @@ public abstract class CopierTemplate4 implements Copier{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CopierTemplate4.class);
private SelectionVector4 sv4;
- private RecordBatch incoming;
private RecordBatch outgoing;
@Override
- public void setupRemover(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, VectorAllocator[] allocators) throws SchemaChangeException{
- this.incoming = incoming;
+ public void setupRemover(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException{
this.outgoing = outgoing;
this.sv4 = incoming.getSelectionVector4();
doSetup(context, incoming, outgoing);
@@ -46,9 +44,8 @@ public abstract class CopierTemplate4 implements Copier{
@Override
public int copyRecords(int index, int recordCount){
-// logger.debug("Copying records.");
for(VectorWrapper<?> out : outgoing){
- out.getValueVector().allocateNewSafe();
+ out.getValueVector().allocateNew();
}
int outgoingPosition = 0;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
index 558ce63..42f2128 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
@@ -195,7 +195,7 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
private List<ValueVector> out = Lists.newArrayList();
@Override
- public void setupRemover(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, VectorAllocator[] allocators){
+ public void setupRemover(FragmentContext context, RecordBatch incoming, RecordBatch outgoing){
for(VectorWrapper<?> vv : incoming){
TransferPair tp = vv.getValueVector().getTransferPair();
pairs.add(tp);
@@ -220,7 +220,7 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
private Copier getStraightCopier(){
StraightCopier copier = new StraightCopier();
- copier.setupRemover(context, incoming, this, null);
+ copier.setupRemover(context, incoming, this);
container.addCollection(copier.getOut());
return copier;
}
@@ -237,7 +237,7 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
final CodeGenerator<Copier> cg = CodeGenerator.get(Copier.TEMPLATE_DEFINITION2, context.getFunctionRegistry());
CopyUtil.generateCopies(cg.getRoot(), incoming, false);
Copier copier = context.getImplementationClass(cg);
- copier.setupRemover(context, incoming, this, null);
+ copier.setupRemover(context, incoming, this);
return copier;
} catch (ClassTransformationException | IOException e) {
@@ -262,7 +262,7 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
final CodeGenerator<Copier> cg = CodeGenerator.get(Copier.TEMPLATE_DEFINITION4, context.getFunctionRegistry());
CopyUtil.generateCopies(cg.getRoot(), batch, true);
Copier copier = context.getImplementationClass(cg);
- copier.setupRemover(context, batch, outgoing, null);
+ copier.setupRemover(context, batch, outgoing);
return copier;
} catch (ClassTransformationException | IOException e) {
@@ -275,15 +275,6 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
return WritableBatch.get(this);
}
- public static VectorAllocator getAllocator4(ValueVector outgoing){
- if(outgoing instanceof FixedWidthVector){
- return new FixedVectorAllocator((FixedWidthVector) outgoing);
- }else if(outgoing instanceof VariableWidthVector ){
- return new VariableEstimatedVector( (VariableWidthVector) outgoing, 250);
- }else{
- throw new UnsupportedOperationException();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
index 0ce480d..cea5460 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
@@ -60,7 +60,8 @@ public class SimpleParallelizer {
private double affinityFactor;
public SimpleParallelizer(QueryContext context){
- this.parallelizationThreshold = context.getOptions().getOption(ExecConstants.SLICE_TARGET).num_val;
+ long sliceTarget = context.getOptions().getOption(ExecConstants.SLICE_TARGET).num_val;
+ this.parallelizationThreshold = sliceTarget > 0 ? sliceTarget : 1;
this.maxWidthPerNode = context.getOptions().getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val.intValue();
this.maxGlobalWidth = context.getOptions().getOption(ExecConstants.MAX_WIDTH_GLOBAL_KEY).num_val.intValue();
this.affinityFactor = context.getOptions().getOption(ExecConstants.AFFINITY_FACTOR_KEY).float_val.intValue();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
index 1f2c33a..49c7399 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
@@ -42,7 +42,7 @@ public class VectorContainer extends AbstractMapVector implements Iterable<Vecto
private BatchSchema schema;
private int recordCount = -1;
private final OperatorContext oContext;
-
+
public VectorContainer() {
this.oContext = null;
}
@@ -74,12 +74,12 @@ public class VectorContainer extends AbstractMapVector implements Iterable<Vecto
add(vv, releasable);
}
- public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz){
+ public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz){
MaterializedField field = MaterializedField.create(name, type);
ValueVector v = TypeHelper.getNewVector(field, this.oContext.getAllocator());
-
+
add(v);
-
+
if(clazz.isAssignableFrom(v.getClass())){
return (T) v;
}else{
@@ -252,5 +252,18 @@ public class VectorContainer extends AbstractMapVector implements Iterable<Vecto
return this.wrappers.size();
}
+ public void allocateNew(){
+ for (VectorWrapper<?> w : wrappers) {
+ w.getValueVector().allocateNew();
+ }
+ }
+
+ public boolean allocateNewSafe(){
+ for (VectorWrapper<?> w : wrappers) {
+ if(!w.getValueVector().allocateNewSafe()) return false;
+ }
+
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3eea13c/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VectorAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VectorAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VectorAllocator.java
index 77b6e1c..eb01bef 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VectorAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VectorAllocator.java
@@ -22,22 +22,23 @@ import org.apache.drill.exec.vector.RepeatedVariableWidthVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VariableWidthVector;
+@Deprecated
public abstract class VectorAllocator{
public abstract void alloc(int recordCount);
-
- public static VectorAllocator getAllocator(ValueVector in, ValueVector outgoing){
- if(outgoing instanceof FixedWidthVector){
- return new FixedVectorAllocator((FixedWidthVector) outgoing);
- }else if(outgoing instanceof VariableWidthVector && in instanceof VariableWidthVector){
- return new VariableVectorAllocator( (VariableWidthVector) in, (VariableWidthVector) outgoing);
- } else if (outgoing instanceof RepeatedVariableWidthVector && in instanceof RepeatedVariableWidthVector) {
- return new RepeatedVectorAllocator((RepeatedVariableWidthVector) in, (RepeatedVariableWidthVector) outgoing);
- }else{
- throw new UnsupportedOperationException();
- }
- }
-
-
+
+// public static VectorAllocator getAllocator(ValueVector in, ValueVector outgoing){
+// if(outgoing instanceof FixedWidthVector){
+// return new FixedVectorAllocator((FixedWidthVector) outgoing);
+// }else if(outgoing instanceof VariableWidthVector && in instanceof VariableWidthVector){
+// return new VariableVectorAllocator( (VariableWidthVector) in, (VariableWidthVector) outgoing);
+// } else if (outgoing instanceof RepeatedVariableWidthVector && in instanceof RepeatedVariableWidthVector) {
+// return new RepeatedVectorAllocator((RepeatedVariableWidthVector) in, (RepeatedVariableWidthVector) outgoing);
+// }else{
+// throw new UnsupportedOperationException();
+// }
+// }
+
+ @Deprecated
public static VectorAllocator getAllocator(ValueVector outgoing, int averageBytesPerVariable){
if(outgoing instanceof FixedWidthVector){
return new FixedVectorAllocator((FixedWidthVector) outgoing);
[11/14] git commit: DRILL-1055: Add ProducerConsumer operator to scans
Posted by ja...@apache.org.
DRILL-1055: Add ProducerConsumer operator to scans
This can be disabled. The queue size is configurable
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/208d511b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/208d511b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/208d511b
Branch: refs/heads/master
Commit: 208d511beebf2369ff453df09843e16f1433fd36
Parents: af29c25
Author: Steven Phillips <sp...@maprtech.com>
Authored: Sat Jun 21 17:11:31 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 25 09:10:12 2014 -0700
----------------------------------------------------------------------
.../physical/base/AbstractPhysicalVisitor.java | 6 +
.../exec/physical/base/PhysicalVisitor.java | 2 +
.../exec/physical/config/ProducerConsumer.java | 58 ++++++
.../impl/producer/ProducerConsumerBatch.java | 199 +++++++++++++++++++
.../producer/ProducerConsumerBatchCreator.java | 36 ++++
.../exec/planner/physical/PlannerSettings.java | 3 +
.../planner/physical/ProducerConsumerPrel.java | 78 ++++++++
.../drill/exec/planner/physical/ScanPrel.java | 2 +-
.../visitor/ProducerConsumerPrelVisitor.java | 59 ++++++
.../planner/sql/handlers/DefaultSqlHandler.java | 13 +-
.../server/options/SystemOptionManager.java | 2 +
.../apache/drill/exec/proto/UserBitShared.java | 16 +-
.../exec/proto/beans/CoreOperatorType.java | 4 +-
protocol/src/main/protobuf/UserBitShared.proto | 1 +
14 files changed, 473 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
index d7d7c12..5e85425 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
@@ -28,6 +28,7 @@ import org.apache.drill.exec.physical.config.Limit;
import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.physical.config.MergingReceiverPOP;
import org.apache.drill.exec.physical.config.OrderedPartitionSender;
+import org.apache.drill.exec.physical.config.ProducerConsumer;
import org.apache.drill.exec.physical.config.Project;
import org.apache.drill.exec.physical.config.UnorderedReceiver;
import org.apache.drill.exec.physical.config.RangeSender;
@@ -186,6 +187,11 @@ public abstract class AbstractPhysicalVisitor<T, X, E extends Throwable> impleme
}
@Override
+ public T visitProducerConsumer(ProducerConsumer op, X value) throws E {
+ return visitOp(op, value);
+ }
+
+ @Override
public T visitIteratorValidator(IteratorValidator op, X value) throws E {
return visitOp(op, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
index 6d5a6cb..f0b0b9a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
@@ -28,6 +28,7 @@ import org.apache.drill.exec.physical.config.Limit;
import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.physical.config.MergingReceiverPOP;
import org.apache.drill.exec.physical.config.OrderedPartitionSender;
+import org.apache.drill.exec.physical.config.ProducerConsumer;
import org.apache.drill.exec.physical.config.Project;
import org.apache.drill.exec.physical.config.UnorderedReceiver;
import org.apache.drill.exec.physical.config.RangeSender;
@@ -79,6 +80,7 @@ public interface PhysicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
public RETURN visitScreen(Screen op, EXTRA value) throws EXCEP;
public RETURN visitSingleSender(SingleSender op, EXTRA value) throws EXCEP;
public RETURN visitUnionExchange(UnionExchange op, EXTRA value) throws EXCEP;
+ public RETURN visitProducerConsumer(ProducerConsumer op, EXTRA value) throws EXCEP;
public RETURN visitIteratorValidator(IteratorValidator op, EXTRA value) throws EXCEP;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ProducerConsumer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ProducerConsumer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ProducerConsumer.java
new file mode 100644
index 0000000..87655d1
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ProducerConsumer.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.config;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.drill.exec.physical.base.AbstractSingle;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
+
+@JsonTypeName("producer-consumer")
+public class ProducerConsumer extends AbstractSingle{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProducerConsumer.class);
+
+ private final int size;
+
+ @JsonCreator
+ public ProducerConsumer(@JsonProperty("child") PhysicalOperator child, @JsonProperty("size") int size) {
+ super(child);
+ this.size = size;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
+ return physicalVisitor.visitProducerConsumer(this, value);
+ }
+
+ @Override
+ protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
+ return new ProducerConsumer(child, size);
+ }
+
+ public int getSize() {
+ return size;
+ }
+
+ @Override
+ public int getOperatorType() {
+ return CoreOperatorType.PRODUCER_CONSUMER_VALUE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
new file mode 100644
index 0000000..9ec07de
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.producer;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.ProducerConsumer;
+import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
+import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
+import org.apache.drill.exec.vector.ValueVector;
+
+import java.util.Queue;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingDeque;
+
+public class ProducerConsumerBatch extends AbstractRecordBatch {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProducerConsumerBatch.class);
+
+ private RecordBatch incoming;
+ private Thread producer = new Thread(new Producer(), Thread.currentThread().getName() + " - Producer Thread");
+ private boolean running = false;
+ private BlockingDeque<RecordBatchDataWrapper> queue;
+ private int recordCount;
+ private BatchSchema schema;
+ private boolean stop = false;
+
+ protected ProducerConsumerBatch(ProducerConsumer popConfig, FragmentContext context, RecordBatch incoming) throws OutOfMemoryException {
+ super(popConfig, context);
+ this.incoming = incoming;
+ this.queue = new LinkedBlockingDeque<>(popConfig.getSize());
+ }
+
+ @Override
+ public IterOutcome innerNext() {
+ if (!running) {
+ producer.start();
+ running = true;
+ }
+ RecordBatchDataWrapper wrapper;
+ try {
+ stats.startWait();
+ wrapper = queue.take();
+ logger.debug("Got batch from queue");
+ } catch (InterruptedException e) {
+ context.fail(e);
+ return IterOutcome.STOP;
+ } finally {
+ stats.stopWait();
+ }
+ if (wrapper.finished) {
+ return IterOutcome.NONE;
+ }
+
+ recordCount = wrapper.batch.getRecordCount();
+ boolean newSchema = load(wrapper.batch);
+
+ return newSchema ? IterOutcome.OK_NEW_SCHEMA : IterOutcome.OK;
+ }
+
+ private boolean load(RecordBatchData batch) {
+ VectorContainer newContainer = batch.getContainer();
+ if (schema != null && newContainer.getSchema().equals(schema)) {
+ container.zeroVectors();
+ BatchSchema schema = container.getSchema();
+ for (int i = 0; i < container.getNumberOfColumns(); i++) {
+ MaterializedField field = schema.getColumn(i);
+ MajorType type = field.getType();
+ ValueVector vOut = container.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()),
+ container.getValueVectorId(field.getPath()).getFieldIds()).getValueVector();
+ ValueVector vIn = newContainer.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()),
+ newContainer.getValueVectorId(field.getPath()).getFieldIds()).getValueVector();
+ TransferPair tp = vIn.makeTransferPair(vOut);
+ tp.transfer();
+ }
+ return false;
+ } else {
+ container.clear();
+ for (VectorWrapper w : newContainer) {
+ container.add(w.getValueVector());
+ }
+ container.buildSchema(SelectionVectorMode.NONE);
+ schema = container.getSchema();
+ return true;
+ }
+ }
+
+ private class Producer implements Runnable {
+
+ @Override
+ public void run() {
+ if (stop) return;
+ outer: while (true) {
+ IterOutcome upstream = incoming.next();
+ switch (upstream) {
+ case NONE:
+ break outer;
+ case STOP:
+ try {
+ queue.putFirst(new RecordBatchDataWrapper(null, false, true));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ case OK_NEW_SCHEMA:
+ case OK:
+ try {
+ if (!stop) queue.put(new RecordBatchDataWrapper(new RecordBatchData(incoming), false, false));
+ } catch (InterruptedException e) {
+ context.fail(e);
+ try {
+ queue.putFirst(new RecordBatchDataWrapper(null, false, true));
+ } catch (InterruptedException e1) {
+ throw new RuntimeException(e1);
+ }
+ }
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+ try {
+ queue.put(new RecordBatchDataWrapper(null, true, false));
+ } catch (InterruptedException e) {
+ context.fail(e);
+ try {
+ queue.putFirst(new RecordBatchDataWrapper(null, false, true));
+ } catch (InterruptedException e1) {
+ throw new RuntimeException(e1);
+ }
+ }
+ logger.debug("Producer thread finished");
+ }
+ }
+
+ private void clearQueue() {
+ RecordBatchDataWrapper wrapper;
+ while ((wrapper = queue.poll()) != null) {
+ wrapper.batch.getContainer().clear();
+ }
+ }
+
+ @Override
+ protected void killIncoming() {
+ incoming.kill();
+ }
+
+ @Override
+ public void cleanup() {
+ stop = true;
+ clearQueue();
+ super.cleanup();
+ incoming.cleanup();
+ }
+
+ @Override
+ public int getRecordCount() {
+ return recordCount;
+ }
+
+ private static class RecordBatchDataWrapper {
+ RecordBatchData batch;
+ boolean finished;
+ boolean failed;
+
+ RecordBatchDataWrapper(RecordBatchData batch, boolean finished, boolean failed) {
+ this.batch = batch;
+ this.finished = finished;
+ this.failed = failed;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatchCreator.java
new file mode 100644
index 0000000..0fcf4f3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatchCreator.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.producer;
+
+import com.google.common.collect.Iterables;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.Limit;
+import org.apache.drill.exec.physical.config.ProducerConsumer;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.physical.impl.producer.ProducerConsumerBatch;
+import org.apache.drill.exec.record.RecordBatch;
+
+import java.util.List;
+
+public class ProducerConsumerBatchCreator implements BatchCreator<ProducerConsumer> {
+ @Override
+ public RecordBatch getBatch(FragmentContext context, ProducerConsumer config, List<RecordBatch> children) throws ExecutionSetupException {
+ return new ProducerConsumerBatch(config, context, Iterables.getOnlyElement(children));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index e10b620..ae0ac32 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -23,6 +23,7 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValidator;
import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator;
+import org.apache.drill.exec.server.options.TypeValidators.LongValidator;
import org.apache.drill.exec.server.options.TypeValidators.PositiveLongValidator;
import org.apache.drill.exec.server.options.TypeValidators.RangeDoubleValidator;
@@ -43,6 +44,8 @@ public class PlannerSettings implements FrameworkContext{
public static final OptionValidator BROADCAST = new BooleanValidator("planner.enable_broadcast_join", true);
public static final OptionValidator BROADCAST_THRESHOLD = new PositiveLongValidator("planner.broadcast_threshold", MAX_BROADCAST_THRESHOLD, 1000000);
public static final OptionValidator JOIN_ROW_COUNT_ESTIMATE_FACTOR = new RangeDoubleValidator("planner.join.row_count_estimate_factor", 0, 100, 1.0d);
+ public static final OptionValidator PRODUCER_CONSUMER = new BooleanValidator("planner.add_producer_consumer", true);
+ public static final OptionValidator PRODUCER_CONSUMER_QUEUE_SIZE = new LongValidator("planner.producer_consumer_queue_size", 10);
public OptionManager options = null;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProducerConsumerPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProducerConsumerPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProducerConsumerPrel.java
new file mode 100644
index 0000000..35aebdc
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProducerConsumerPrel.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical;
+
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.ProducerConsumer;
+import org.apache.drill.exec.planner.common.DrillRelNode;
+import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.SingleRel;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelTraitSet;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+public class ProducerConsumerPrel extends SingleRel implements DrillRelNode, Prel {
+
+ private int queueSize;
+
+ public ProducerConsumerPrel(RelNode child, int queueSize) {
+ super(child.getCluster(), child.getTraitSet(), child);
+ this.queueSize = queueSize;
+ }
+
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ return new ProducerConsumerPrel(sole(inputs), queueSize);
+ }
+
+ @Override
+ public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
+ Prel child = (Prel) this.getChild();
+
+ PhysicalOperator childPOP = child.getPhysicalOperator(creator);
+
+ ProducerConsumer pop = new ProducerConsumer(childPOP, queueSize);
+
+ return creator.addMetadata(this, pop);
+ }
+
+ @Override
+ public Iterator<Prel> iterator() {
+ return PrelUtil.iter(getChild());
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PrelVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitPrel(this, value);
+ }
+
+ @Override
+ public SelectionVectorMode[] getSupportedEncodings() {
+ return SelectionVectorMode.DEFAULT;
+ }
+
+ @Override
+ public SelectionVectorMode getEncoding() {
+ return SelectionVectorMode.NONE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
index 9bc71c0..fc62331 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
@@ -134,7 +134,7 @@ public class ScanPrel extends AbstractRelNode implements DrillScanPrel {
@Override
public <T, X, E extends Throwable> T accept(PrelVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitPrel(this, value);
+ return logicalVisitor.visitScan(this, value);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ProducerConsumerPrelVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ProducerConsumerPrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ProducerConsumerPrelVisitor.java
new file mode 100644
index 0000000..575b4b4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ProducerConsumerPrelVisitor.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical.visitor;
+
+import com.google.common.collect.Lists;
+import org.apache.drill.exec.planner.physical.Prel;
+import org.apache.drill.exec.planner.physical.ProducerConsumerPrel;
+import org.apache.drill.exec.planner.physical.ScanPrel;
+import org.apache.drill.exec.planner.physical.SelectionVectorRemoverPrel;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.eigenbase.rel.RelNode;
+
+import java.util.List;
+
+
+public class ProducerConsumerPrelVisitor extends BasePrelVisitor<Prel, Void, RuntimeException>{
+
+ private int queueSize;
+
+ public static Prel addProducerConsumerToScans(Prel prel, int queueSize){
+
+ return prel.accept(new ProducerConsumerPrelVisitor(queueSize), null);
+ }
+
+ public ProducerConsumerPrelVisitor(int queueSize) {
+ this.queueSize = queueSize;
+ }
+
+ @Override
+ public Prel visitPrel(Prel prel, Void value) throws RuntimeException {
+ List<RelNode> children = Lists.newArrayList();
+ for(Prel child : prel){
+ children.add(child.accept(this, null));
+ }
+ return (Prel) prel.copy(prel.getTraitSet(), children);
+ }
+
+ @Override
+ public Prel visitScan(ScanPrel scanPrel, Void value) {
+ return new ProducerConsumerPrel(scanPrel, queueSize);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index 21420df..14db66c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -40,12 +40,14 @@ import org.apache.drill.exec.planner.logical.DrillScreenRel;
import org.apache.drill.exec.planner.logical.DrillStoreRel;
import org.apache.drill.exec.planner.physical.DrillDistributionTrait;
import org.apache.drill.exec.planner.physical.PhysicalPlanCreator;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.planner.physical.explain.PrelSequencer;
import org.apache.drill.exec.planner.physical.visitor.ExcessiveExchangeIdentifier;
import org.apache.drill.exec.planner.physical.visitor.FinalColumnReorderer;
import org.apache.drill.exec.planner.physical.visitor.ComplexToJsonPrelVisitor;
import org.apache.drill.exec.planner.physical.visitor.JoinPrelRenameVisitor;
+import org.apache.drill.exec.planner.physical.visitor.ProducerConsumerPrelVisitor;
import org.apache.drill.exec.planner.physical.visitor.RelUniqifier;
import org.apache.drill.exec.planner.physical.visitor.SelectionVectorPrelVisitor;
import org.apache.drill.exec.planner.sql.DrillSqlWorker;
@@ -178,6 +180,15 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
phyRelNode = SelectionVectorPrelVisitor.addSelectionRemoversWhereNecessary(phyRelNode);
/* 5.)
+ * Add ProducerConsumer after each scan if the option is set
+ * Use the configured queueSize
+ */
+ if (context.getOptions().getOption(PlannerSettings.PRODUCER_CONSUMER.getOptionName()).bool_val) {
+ long queueSize = context.getOptions().getOption(PlannerSettings.PRODUCER_CONSUMER_QUEUE_SIZE.getOptionName()).num_val;
+ phyRelNode = ProducerConsumerPrelVisitor.addProducerConsumerToScans(phyRelNode, (int) queueSize);
+ }
+
+ /* 6.)
* if the client does not support complex types (Map, Repeated)
* insert a project which which would convert
*/
@@ -186,7 +197,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
phyRelNode = ComplexToJsonPrelVisitor.addComplexToJsonPrel(phyRelNode);
}
- /* 6.)
+ /* 7.)
* Finally, Make sure that the no rels are repeats.
* This could happen in the case of querying the same table twice as Optiq may canonicalize these.
*/
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index a42640f..424d7ff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -54,6 +54,8 @@ public class SystemOptionManager implements OptionManager{
PlannerSettings.BROADCAST,
PlannerSettings.BROADCAST_THRESHOLD,
PlannerSettings.JOIN_ROW_COUNT_ESTIMATE_FACTOR,
+ PlannerSettings.PRODUCER_CONSUMER,
+ PlannerSettings.PRODUCER_CONSUMER_QUEUE_SIZE,
ExecConstants.OUTPUT_FORMAT_VALIDATOR,
ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,
ExecConstants.SLICE_TARGET_OPTION,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
index 2f4a58c..64cbcbf 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
@@ -458,6 +458,10 @@ public final class UserBitShared {
* <code>COMPLEX_TO_JSON = 31;</code>
*/
COMPLEX_TO_JSON(31, 31),
+ /**
+ * <code>PRODUCER_CONSUMER = 32;</code>
+ */
+ PRODUCER_CONSUMER(32, 32),
;
/**
@@ -588,6 +592,10 @@ public final class UserBitShared {
* <code>COMPLEX_TO_JSON = 31;</code>
*/
public static final int COMPLEX_TO_JSON_VALUE = 31;
+ /**
+ * <code>PRODUCER_CONSUMER = 32;</code>
+ */
+ public static final int PRODUCER_CONSUMER_VALUE = 32;
public final int getNumber() { return value; }
@@ -626,6 +634,7 @@ public final class UserBitShared {
case 29: return JSON_SUB_SCAN;
case 30: return INFO_SCHEMA_SUB_SCAN;
case 31: return COMPLEX_TO_JSON;
+ case 32: return PRODUCER_CONSUMER;
default: return null;
}
}
@@ -16530,7 +16539,7 @@ public final class UserBitShared {
"\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003*k\n\rFragmen" +
"tState\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATI",
"ON\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCE" +
- "LLED\020\004\022\n\n\006FAILED\020\005*\372\004\n\020CoreOperatorType\022" +
+ "LLED\020\004\022\n\n\006FAILED\020\005*\221\005\n\020CoreOperatorType\022" +
"\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001" +
"\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH" +
"_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITIO" +
@@ -16546,8 +16555,9 @@ public final class UserBitShared {
"UET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEX" +
"T_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SU" +
"B_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017CO" +
- "MPLEX_TO_JSON\020\037B.\n\033org.apache.drill.exec" +
- ".protoB\rUserBitSharedH\001"
+ "MPLEX_TO_JSON\020\037\022\025\n\021PRODUCER_CONSUMER\020 B." +
+ "\n\033org.apache.drill.exec.protoB\rUserBitSh" +
+ "aredH\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
index 0485a95..e8039c9 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
@@ -53,7 +53,8 @@ public enum CoreOperatorType implements com.dyuproject.protostuff.EnumLite<CoreO
TEXT_SUB_SCAN(28),
JSON_SUB_SCAN(29),
INFO_SCHEMA_SUB_SCAN(30),
- COMPLEX_TO_JSON(31);
+ COMPLEX_TO_JSON(31),
+ PRODUCER_CONSUMER(32);
public final int number;
@@ -103,6 +104,7 @@ public enum CoreOperatorType implements com.dyuproject.protostuff.EnumLite<CoreO
case 29: return JSON_SUB_SCAN;
case 30: return INFO_SCHEMA_SUB_SCAN;
case 31: return COMPLEX_TO_JSON;
+ case 32: return PRODUCER_CONSUMER;
default: return null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/208d511b/protocol/src/main/protobuf/UserBitShared.proto
----------------------------------------------------------------------
diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto
index 6fd73db..10dce1f 100644
--- a/protocol/src/main/protobuf/UserBitShared.proto
+++ b/protocol/src/main/protobuf/UserBitShared.proto
@@ -199,4 +199,5 @@ enum CoreOperatorType {
JSON_SUB_SCAN = 29;
INFO_SCHEMA_SUB_SCAN = 30;
COMPLEX_TO_JSON = 31;
+ PRODUCER_CONSUMER = 32;
}