You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pa...@apache.org on 2017/02/08 06:23:59 UTC

[1/4] drill git commit: DRILL-5223: Drill should ensure balanced workload assignment at node level in order to get better query performance

Repository: drill
Updated Branches:
  refs/heads/master ddcf89548 -> 2b5f5428a


DRILL-5223: Drill should ensure balanced workload assignment at node level in order to get better query performance

This closes #730


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e386e5e1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e386e5e1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e386e5e1

Branch: refs/heads/master
Commit: e386e5e145945f9644184c642b94f21438215762
Parents: ddcf895
Author: Padma Penumarthy <pp...@yahoo.com>
Authored: Fri Jan 20 17:57:10 2017 -0800
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Tue Feb 7 22:21:52 2017 -0800

----------------------------------------------------------------------
 .../exec/store/schedule/AssignmentCreator.java  | 25 ++++++++-
 .../drill/exec/store/store/TestAssignment.java  | 58 +++++++++++++++++++-
 2 files changed, 79 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/e386e5e1/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
index 127264a..aeaf4bf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
@@ -106,9 +106,13 @@ public class AssignmentCreator<T extends CompleteWork> {
     LinkedList<WorkEndpointListPair<T>> unassignedWorkList;
     Map<DrillbitEndpoint,FragIteratorWrapper> endpointIterators = getEndpointIterators();
 
+    // Assign upto minCount per node based on locality.
     unassignedWorkList = assign(workList, endpointIterators, true);
-
+    // Assign upto maxCount per node based on locality.
+    unassignedWorkList = assign(unassignedWorkList, endpointIterators, false);
+    // Assign upto minCount per node in a round robin fashion.
     assignLeftovers(unassignedWorkList, endpointIterators, true);
+    // Assign upto maxCount per node in a round robin fashion.
     assignLeftovers(unassignedWorkList, endpointIterators, false);
 
     if (unassignedWorkList.size() != 0) {
@@ -241,13 +245,30 @@ public class AssignmentCreator<T extends CompleteWork> {
       mmap.put(endpoint, intList);
     }
 
+    int totalMaxCount = 0;
     for (DrillbitEndpoint endpoint : mmap.keySet()) {
       FragIteratorWrapper wrapper = new FragIteratorWrapper();
       wrapper.iter = Iterators.cycle(mmap.get(endpoint));
-      wrapper.maxCount = maxWork * mmap.get(endpoint).size();
+      // To distribute the load among nodes equally, limit the maxCount per node.
+      int maxCount = (int) ((double)mmap.get(endpoint).size()/incomingEndpoints.size() * units.size());
+      wrapper.maxCount = Math.min(maxWork * mmap.get(endpoint).size(), maxCount);
+      totalMaxCount += wrapper.maxCount;
       wrapper.minCount = Math.max(maxWork - 1, 1) * mmap.get(endpoint).size();
       map.put(endpoint, wrapper);
     }
+
+    // Take care of leftovers.
+    while (totalMaxCount < units.size()) {
+      for (Entry<DrillbitEndpoint, FragIteratorWrapper> entry : map.entrySet()) {
+        FragIteratorWrapper iteratorWrapper = entry.getValue();
+        iteratorWrapper.maxCount++;
+        totalMaxCount++;
+        if (totalMaxCount == units.size()) {
+          break;
+        }
+      }
+    }
+
     return map;
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/e386e5e1/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestAssignment.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestAssignment.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestAssignment.java
index 65d8cf7..aceaf79 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestAssignment.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestAssignment.java
@@ -39,18 +39,72 @@ public class TestAssignment {
 
   private static final long FILE_SIZE = 1000;
   private static List<DrillbitEndpoint> endpoints;
+  private static final int numEndPoints = 30;
+  private final int widthPerNode = 23;
 
   @BeforeClass
   public static void setup() {
     endpoints = Lists.newArrayList();
     final String pattern = "node%d";
-    for (int i = 2; i < 32; i++) {
+    for (int i = 0; i < numEndPoints; i++) {
       String host = String.format(pattern, i);
       endpoints.add(DrillbitEndpoint.newBuilder().setAddress(host).build());
     }
   }
 
   @Test
+  public void testBalanceAcrossNodes() throws Exception {
+    int numChunks = widthPerNode * numEndPoints + 100;
+    List<CompleteFileWork> chunks = generateChunks(numChunks);
+    Iterator<DrillbitEndpoint> incomingEndpointsIterator = Iterators.cycle(endpoints);
+    List<DrillbitEndpoint> incomingEndpoints = Lists.newArrayList();
+    List<Integer> expectedAssignments = Lists.newArrayList();
+    List<Integer> actualAssignments = Lists.newArrayList();
+
+    final int width = widthPerNode * numEndPoints;
+    for (int i = 0; i < width; i++) {
+      incomingEndpoints.add(incomingEndpointsIterator.next());
+    }
+
+    // Calculate expected assignments for each node.
+    final int numAssignmentsPerNode = numChunks/numEndPoints;
+    int leftOver = numChunks - numAssignmentsPerNode * numEndPoints;
+    for (int i =0; i < numEndPoints; i++) {
+      int additional = leftOver > 0 ? 1 : 0;
+      expectedAssignments.add(numAssignmentsPerNode + additional);
+      if (leftOver > 0) {
+        leftOver--;
+      }
+    }
+
+    ListMultimap<Integer, CompleteFileWork> mappings = AssignmentCreator.getMappings(incomingEndpoints, chunks);
+    System.out.println(mappings.keySet().size());
+
+    // Verify that all fragments have chunks assigned.
+    for (int i = 0; i < width; i++) {
+      Assert.assertTrue("no mapping for entry " + i, mappings.get(i) != null && mappings.get(i).size() > 0);
+    }
+
+    // Verify actual and expected assignments per node match.
+    // Compute actual assignments for each node.
+    for (int i=0; i < numEndPoints; i++) {
+      int numAssignments = 0;
+      int index = i;
+
+      while(index < numEndPoints * widthPerNode) {
+        numAssignments += mappings.get(index).size();
+        index += numEndPoints;
+      }
+
+      actualAssignments.add(numAssignments);
+    }
+
+    for (int i=0; i < numEndPoints; i++) {
+      Assert.assertTrue(actualAssignments.get(i) == expectedAssignments.get(i));
+    }
+  }
+
+  @Test
   public void manyFiles() throws Exception {
     List<CompleteFileWork> chunks = generateChunks(1000);
 
@@ -58,7 +112,7 @@ public class TestAssignment {
 
     List<DrillbitEndpoint> incomingEndpoints = Lists.newArrayList();
 
-    final int width = 28 * 30;
+    final int width = widthPerNode * numEndPoints;
     for (int i = 0; i < width; i++) {
       incomingEndpoints.add(incomingEndpointsIterator.next());
     }


[4/4] drill git commit: DRILL-5219: Relax user properties validation in C++ client

Posted by pa...@apache.org.
DRILL-5219: Relax user properties validation in C++ client

Unlike Java client, C++ client only allows user properties present in a
whitelist. Relax this restriction so that user can add extra properties.

This closes #727


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2b5f5428
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2b5f5428
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2b5f5428

Branch: refs/heads/master
Commit: 2b5f5428af6951c71461c2bd2253e184ad0c4631
Parents: 478de24
Author: Laurent Goujon <la...@dremio.com>
Authored: Wed Jan 25 10:32:33 2017 -0800
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Tue Feb 7 22:22:53 2017 -0800

----------------------------------------------------------------------
 contrib/native/client/src/clientlib/drillClient.cpp     | 6 +-----
 contrib/native/client/src/clientlib/drillClientImpl.cpp | 9 +++++++--
 2 files changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/2b5f5428/contrib/native/client/src/clientlib/drillClient.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClient.cpp b/contrib/native/client/src/clientlib/drillClient.cpp
index b456d17..f97a25c 100644
--- a/contrib/native/client/src/clientlib/drillClient.cpp
+++ b/contrib/native/client/src/clientlib/drillClient.cpp
@@ -367,11 +367,7 @@ connectionStatus_t DrillClient::connect(const char* connectStr, DrillUserPropert
     connectionStatus_t ret=CONN_SUCCESS;
     ret=this->m_pImpl->connect(connectStr);
     if(ret==CONN_SUCCESS){
-        if(properties!=NULL){
-            ret=this->m_pImpl->validateHandshake(properties);
-        }else{
-            ret=this->m_pImpl->validateHandshake(NULL);
-        }
+        ret=this->m_pImpl->validateHandshake(properties);
     }
     return ret;
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/2b5f5428/contrib/native/client/src/clientlib/drillClientImpl.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClientImpl.cpp b/contrib/native/client/src/clientlib/drillClientImpl.cpp
index 8794044..05171e5 100644
--- a/contrib/native/client/src/clientlib/drillClientImpl.cpp
+++ b/contrib/native/client/src/clientlib/drillClientImpl.cpp
@@ -370,8 +370,13 @@ connectionStatus_t DrillClientImpl::validateHandshake(DrillUserProperties* prope
         for(size_t i=0; i<properties->size(); i++){
             std::map<std::string,uint32_t>::const_iterator it=DrillUserProperties::USER_PROPERTIES.find(properties->keyAt(i));
             if(it==DrillUserProperties::USER_PROPERTIES.end()){
-                DRILL_MT_LOG(DRILL_LOG(LOG_WARNING) << "Connection property ("<< properties->keyAt(i)
-                    << ") is unknown and is being skipped" << std::endl;)
+                DRILL_MT_LOG(DRILL_LOG(LOG_INFO) << "Connection property ("<< properties->keyAt(i)
+                    << ") is unknown" << std::endl;)
+
+                exec::user::Property* connProp = userProperties->add_properties();
+                connProp->set_key(properties->keyAt(i));
+                connProp->set_value(properties->valueAt(i));
+
                 continue;
             }
             if(IS_BITSET((*it).second,USERPROP_FLAGS_SERVERPROP)){


[3/4] drill git commit: DRILL-5243: Fix TestContextFunctions.sessionIdUDFWithinSameSession unit test

Posted by pa...@apache.org.
DRILL-5243: Fix TestContextFunctions.sessionIdUDFWithinSameSession unit test

This closes #743


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/478de241
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/478de241
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/478de241

Branch: refs/heads/master
Commit: 478de241dd28b41bcb4487fe67937dca33522dc7
Parents: 07b418a
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Mon Feb 6 13:51:49 2017 +0000
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Tue Feb 7 22:22:26 2017 -0800

----------------------------------------------------------------------
 .../java/org/apache/drill/exec/fn/impl/TestContextFunctions.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/478de241/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
index e4e96fd..6ddf2ba 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
@@ -93,7 +93,7 @@ public class TestContextFunctions extends BaseTestQuery {
 
   @Test
   public void sessionIdUDFWithinSameSession() throws Exception {
-    final String sessionIdQuery = "select session_id as sessionId from (values(1));";
+    final String sessionIdQuery = "select session_id as sessionId from (values(1))";
     testBuilder()
         .sqlQuery(sessionIdQuery)
         .ordered()


[2/4] drill git commit: DRILL-5241: JDBC proxy driver: Do not put null value in map

Posted by pa...@apache.org.
DRILL-5241: JDBC proxy driver: Do not put null value in map

This closes #724


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/07b418ad
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/07b418ad
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/07b418ad

Branch: refs/heads/master
Commit: 07b418adb6241e58a6de1ab52468e71b2c1435f3
Parents: e386e5e
Author: David Haller <da...@mailbox.org>
Authored: Fri Jan 20 21:22:52 2017 +0100
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Tue Feb 7 22:22:18 2017 -0800

----------------------------------------------------------------------
 .../src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/07b418ad/exec/jdbc/src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java
index 7d2da53..c014ebf 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/proxy/ProxiesManager.java
@@ -54,7 +54,7 @@ class ProxiesManager {
       Class<Proxy> newProxyReturnClass =
           (Class<Proxy>) Proxy.getProxyClass( interfaceType.getClassLoader(),
                                               new Class[] { interfaceType });
-      interfacesToProxyClassesMap.put( interfaceType, proxyReturnClass );
+      interfacesToProxyClassesMap.put( interfaceType, newProxyReturnClass );
       proxyReturnClass = newProxyReturnClass;
     }
     return proxyReturnClass;