You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pa...@apache.org on 2017/02/08 06:23:59 UTC
[1/4] drill git commit: DRILL-5223: Drill should ensure balanced
workload assignment at node level in order to get better query performance
Repository: drill
Updated Branches:
refs/heads/master ddcf89548 -> 2b5f5428a
DRILL-5223: Drill should ensure balanced workload assignment at node level in order to get better query performance
This closes #730
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e386e5e1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e386e5e1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e386e5e1
Branch: refs/heads/master
Commit: e386e5e145945f9644184c642b94f21438215762
Parents: ddcf895
Author: Padma Penumarthy <pp...@yahoo.com>
Authored: Fri Jan 20 17:57:10 2017 -0800
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Tue Feb 7 22:21:52 2017 -0800
----------------------------------------------------------------------
.../exec/store/schedule/AssignmentCreator.java | 25 ++++++++-
.../drill/exec/store/store/TestAssignment.java | 58 +++++++++++++++++++-
2 files changed, 79 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/e386e5e1/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
index 127264a..aeaf4bf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
@@ -106,9 +106,13 @@ public class AssignmentCreator<T extends CompleteWork> {
LinkedList<WorkEndpointListPair<T>> unassignedWorkList;
Map<DrillbitEndpoint,FragIteratorWrapper> endpointIterators = getEndpointIterators();
+ // Assign upto minCount per node based on locality.
unassignedWorkList = assign(workList, endpointIterators, true);
-
+ // Assign upto maxCount per node based on locality.
+ unassignedWorkList = assign(unassignedWorkList, endpointIterators, false);
+ // Assign upto minCount per node in a round robin fashion.
assignLeftovers(unassignedWorkList, endpointIterators, true);
+ // Assign upto maxCount per node in a round robin fashion.
assignLeftovers(unassignedWorkList, endpointIterators, false);
if (unassignedWorkList.size() != 0) {
@@ -241,13 +245,30 @@ public class AssignmentCreator<T extends CompleteWork> {
mmap.put(endpoint, intList);
}
+ int totalMaxCount = 0;
for (DrillbitEndpoint endpoint : mmap.keySet()) {
FragIteratorWrapper wrapper = new FragIteratorWrapper();
wrapper.iter = Iterators.cycle(mmap.get(endpoint));
- wrapper.maxCount = maxWork * mmap.get(endpoint).size();
+ // To distribute the load among nodes equally, limit the maxCount per node.
+ int maxCount = (int) ((double)mmap.get(endpoint).size()/incomingEndpoints.size() * units.size());
+ wrapper.maxCount = Math.min(maxWork * mmap.get(endpoint).size(), maxCount);
+ totalMaxCount += wrapper.maxCount;
wrapper.minCount = Math.max(maxWork - 1, 1) * mmap.get(endpoint).size();
map.put(endpoint, wrapper);
}
+
+ // Take care of leftovers.
+ while (totalMaxCount < units.size()) {
+ for (Entry<DrillbitEndpoint, FragIteratorWrapper> entry : map.entrySet()) {
+ FragIteratorWrapper iteratorWrapper = entry.getValue();
+ iteratorWrapper.maxCount++;
+ totalMaxCount++;
+ if (totalMaxCount == units.size()) {
+ break;
+ }
+ }
+ }
+
return map;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/e386e5e1/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestAssignment.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestAssignment.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestAssignment.java
index 65d8cf7..aceaf79 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestAssignment.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestAssignment.java
@@ -39,18 +39,72 @@ public class TestAssignment {
private static final long FILE_SIZE = 1000;
private static List<DrillbitEndpoint> endpoints;
+ private static final int numEndPoints = 30;
+ private final int widthPerNode = 23;
@BeforeClass
public static void setup() {
endpoints = Lists.newArrayList();
final String pattern = "node%d";
- for (int i = 2; i < 32; i++) {
+ for (int i = 0; i < numEndPoints; i++) {
String host = String.format(pattern, i);
endpoints.add(DrillbitEndpoint.newBuilder().setAddress(host).build());
}
}
@Test
+ public void testBalanceAcrossNodes() throws Exception {
+ int numChunks = widthPerNode * numEndPoints + 100;
+ List<CompleteFileWork> chunks = generateChunks(numChunks);
+ Iterator<DrillbitEndpoint> incomingEndpointsIterator = Iterators.cycle(endpoints);
+ List<DrillbitEndpoint> incomingEndpoints = Lists.newArrayList();
+ List<Integer> expectedAssignments = Lists.newArrayList();
+ List<Integer> actualAssignments = Lists.newArrayList();
+
+ final int width = widthPerNode * numEndPoints;
+ for (int i = 0; i < width; i++) {
+ incomingEndpoints.add(incomingEndpointsIterator.next());
+ }
+
+ // Calculate expected assignments for each node.
+ final int numAssignmentsPerNode = numChunks/numEndPoints;
+ int leftOver = numChunks - numAssignmentsPerNode * numEndPoints;
+ for (int i =0; i < numEndPoints; i++) {
+ int additional = leftOver > 0 ? 1 : 0;
+ expectedAssignments.add(numAssignmentsPerNode + additional);
+ if (leftOver > 0) {
+ leftOver--;
+ }
+ }
+
+ ListMultimap<Integer, CompleteFileWork> mappings = AssignmentCreator.getMappings(incomingEndpoints, chunks);
+ System.out.println(mappings.keySet().size());
+
+ // Verify that all fragments have chunks assigned.
+ for (int i = 0; i < width; i++) {
+ Assert.assertTrue("no mapping for entry " + i, mappings.get(i) != null && mappings.get(i).size() > 0);
+ }
+
+ // Verify actual and expected assignments per node match.
+ // Compute actual assignments for each node.
+ for (int i=0; i < numEndPoints; i++) {
+ int numAssignments = 0;
+ int index = i;
+
+ while(index < numEndPoints * widthPerNode) {
+ numAssignments += mappings.get(index).size();
+ index += numEndPoints;
+ }
+
+ actualAssignments.add(numAssignments);
+ }
+
+ for (int i=0; i < numEndPoints; i++) {
+ Assert.assertTrue(actualAssignments.get(i) == expectedAssignments.get(i));
+ }
+ }
+
+ @Test
public void manyFiles() throws Exception {
List<CompleteFileWork> chunks = generateChunks(1000);
@@ -58,7 +112,7 @@ public class TestAssignment {
List<DrillbitEndpoint> incomingEndpoints = Lists.newArrayList();
- final int width = 28 * 30;
+ final int width = widthPerNode * numEndPoints;
for (int i = 0; i < width; i++) {
incomingEndpoints.add(incomingEndpointsIterator.next());
}
[4/4] drill git commit: DRILL-5219: Relax user properties validation
in C++ client
Posted by pa...@apache.org.
DRILL-5219: Relax user properties validation in C++ client
Unlike Java client, C++ client only allows user properties present in a
whitelist. Relax this restriction so that user can add extra properties.
This closes #727
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2b5f5428
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2b5f5428
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2b5f5428
Branch: refs/heads/master
Commit: 2b5f5428af6951c71461c2bd2253e184ad0c4631
Parents: 478de24
Author: Laurent Goujon <la...@dremio.com>
Authored: Wed Jan 25 10:32:33 2017 -0800
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Tue Feb 7 22:22:53 2017 -0800
----------------------------------------------------------------------
contrib/native/client/src/clientlib/drillClient.cpp | 6 +-----
contrib/native/client/src/clientlib/drillClientImpl.cpp | 9 +++++++--
2 files changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/2b5f5428/contrib/native/client/src/clientlib/drillClient.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClient.cpp b/contrib/native/client/src/clientlib/drillClient.cpp
index b456d17..f97a25c 100644
--- a/contrib/native/client/src/clientlib/drillClient.cpp
+++ b/contrib/native/client/src/clientlib/drillClient.cpp
@@ -367,11 +367,7 @@ connectionStatus_t DrillClient::connect(const char* connectStr, DrillUserPropert
connectionStatus_t ret=CONN_SUCCESS;
ret=this->m_pImpl->connect(connectStr);
if(ret==CONN_SUCCESS){
- if(properties!=NULL){
- ret=this->m_pImpl->validateHandshake(properties);
- }else{
- ret=this->m_pImpl->validateHandshake(NULL);
- }
+ ret=this->m_pImpl->validateHandshake(properties);
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/2b5f5428/contrib/native/client/src/clientlib/drillClientImpl.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClientImpl.cpp b/contrib/native/client/src/clientlib/drillClientImpl.cpp
index 8794044..05171e5 100644
--- a/contrib/native/client/src/clientlib/drillClientImpl.cpp
+++ b/contrib/native/client/src/clientlib/drillClientImpl.cpp
@@ -370,8 +370,13 @@ connectionStatus_t DrillClientImpl::validateHandshake(DrillUserProperties* prope
for(size_t i=0; i<properties->size(); i++){
std::map<std::string,uint32_t>::const_iterator it=DrillUserProperties::USER_PROPERTIES.find(properties->keyAt(i));
if(it==DrillUserProperties::USER_PROPERTIES.end()){
- DRILL_MT_LOG(DRILL_LOG(LOG_WARNING) << "Connection property ("<< properties->keyAt(i)
- << ") is unknown and is being skipped" << std::endl;)
+ DRILL_MT_LOG(DRILL_LOG(LOG_INFO) << "Connection property ("<< properties->keyAt(i)
+ << ") is unknown" << std::endl;)
+
+ exec::user::Property* connProp = userProperties->add_properties();
+ connProp->set_key(properties->keyAt(i));
+ connProp->set_value(properties->valueAt(i));
+
continue;
}
if(IS_BITSET((*it).second,USERPROP_FLAGS_SERVERPROP)){
[3/4] drill git commit: DRILL-5243: Fix
TestContextFunctions.sessionIdUDFWithinSameSession unit test
Posted by pa...@apache.org.
DRILL-5243: Fix TestContextFunctions.sessionIdUDFWithinSameSession unit test
This closes #743
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/478de241
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/478de241
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/478de241
Branch: refs/heads/master
Commit: 478de241dd28b41bcb4487fe67937dca33522dc7
Parents: 07b418a
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Mon Feb 6 13:51:49 2017 +0000
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Tue Feb 7 22:22:26 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/drill/exec/fn/impl/TestContextFunctions.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/478de241/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
index e4e96fd..6ddf2ba 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
@@ -93,7 +93,7 @@ public class TestContextFunctions extends BaseTestQuery {
@Test
public void sessionIdUDFWithinSameSession() throws Exception {
- final String sessionIdQuery = "select session_id as sessionId from (values(1));";
+ final String sessionIdQuery = "select session_id as sessionId from (values(1))";
testBuilder()
.sqlQuery(sessionIdQuery)
.ordered()
[2/4] drill git commit: DRILL-5241: JDBC proxy driver: Do not put
null value in map
Posted by pa...@apache.org.
DRILL-5241: JDBC proxy driver: Do not put null value in map
This closes #724
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/07b418ad
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/07b418ad
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/07b418ad
Branch: refs/heads/master
Commit: 07b418adb6241e58a6de1ab52468e71b2c1435f3
Parents: e386e5e
Author: David Haller <da...@mailbox.org>
Authored: Fri Jan 20 21:22:52 2017 +0100
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Tue Feb 7 22:22:18 2017 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/07b418ad/exec/jdbc/src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java
index 7d2da53..c014ebf 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java
@@ -54,7 +54,7 @@ class ProxiesManager {
Class<Proxy> newProxyReturnClass =
(Class<Proxy>) Proxy.getProxyClass( interfaceType.getClassLoader(),
new Class[] { interfaceType });
- interfacesToProxyClassesMap.put( interfaceType, proxyReturnClass );
+ interfacesToProxyClassesMap.put( interfaceType, newProxyReturnClass );
proxyReturnClass = newProxyReturnClass;
}
return proxyReturnClass;