You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2017/03/03 22:10:21 UTC
[1/3] drill git commit: DRILL-5208: Finding path to java executable
should be deterministic
Repository: drill
Updated Branches:
refs/heads/master 7ebb985ed -> 3dfb49729
DRILL-5208: Finding path to java executable should be deterministic
See DRILL-5208 for background. Instead of using \u201cfind\u201d to locate the
java command, we use the any information available, resorting to find
only if the \u201cusual suspects\u201d fails. The result is that we use the JDK
java when available, instead of randomly choosing JDK or JRE java.
close #763
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/d96914f1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/d96914f1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/d96914f1
Branch: refs/heads/master
Commit: d96914f161e241902b0d3c204164cedc97471d12
Parents: 7ebb985
Author: Paul Rogers <pr...@maprtech.com>
Authored: Fri Feb 24 14:53:23 2017 -0800
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Thu Mar 2 23:12:45 2017 -0800
----------------------------------------------------------------------
distribution/src/resources/drill-config.sh | 28 ++++++++++++++++++-------
1 file changed, 21 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/d96914f1/distribution/src/resources/drill-config.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh
index ed03b28..1946275 100644
--- a/distribution/src/resources/drill-config.sh
+++ b/distribution/src/resources/drill-config.sh
@@ -347,10 +347,17 @@ case "`uname`" in
CYGWIN*) is_cygwin=true;;
esac
+if $is_cygwin; then
+ JAVA_BIN="java.exe"
+else
+ JAVA_BIN="java"
+fi
+
# Test for or find JAVA_HOME
+
if [ -z "$JAVA_HOME" ]; then
- if [ -e `which java` ]; then
- SOURCE=`which java`
+ SOURCE=`which java`
+ if [ -e $SOURCE ]; then
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
@@ -359,6 +366,7 @@ if [ -z "$JAVA_HOME" ]; then
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
done
JAVA_HOME="$( cd -P "$( dirname "$SOURCE" )" && cd .. && pwd )"
+ JAVA=$SOURCE
fi
# if we didn't set it
if [ -z "$JAVA_HOME" ]; then
@@ -367,12 +375,18 @@ if [ -z "$JAVA_HOME" ]; then
fi
# Now, verify that 'java' binary exists and is suitable for Drill.
-if $is_cygwin; then
- JAVA_BIN="java.exe"
-else
- JAVA_BIN="java"
+# If we started with `which java` above, use that path (after replacing
+# symlinks.) If we started with JAVA_HOME, try in bin. Doing so handles
+# the case in which JAVA_HOME is a JDK that has a nested JRE; we prefer
+# the JDK bin. Finally, if nothing else works, just search for the
+# executable.
+
+if [ -z "$JAVA" ]; then
+ JAVA="$JAVA_HOME/bin/$JAVA_BIN"
+ if [[ ! -e $JAVA ]]; then
+ JAVA=`find -L "$JAVA_HOME" -name $JAVA_BIN -type f | head -n 1`
+ fi
fi
-JAVA=`find -L "$JAVA_HOME" -name $JAVA_BIN -type f | head -n 1`
if [ ! -e "$JAVA" ]; then
fatal_error "Java not found at JAVA_HOME=$JAVA_HOME."
fi
[3/3] drill git commit: DRILL-5293: Change seed for distribution hash
function to differ from that of the hash table
Posted by jn...@apache.org.
DRILL-5293: Change seed for distribution hash function to differ from that of the hash table
close #765
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3dfb4972
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3dfb4972
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3dfb4972
Branch: refs/heads/master
Commit: 3dfb497293a177164a158b246000ee8291ef258c
Parents: f0057e7
Author: Boaz Ben-Zvi <bo...@BBenZvi-E754-MBP13.local>
Authored: Fri Feb 24 16:48:42 2017 -0800
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Fri Mar 3 10:20:37 2017 -0800
----------------------------------------------------------------------
.../drill/exec/planner/physical/HashPrelUtil.java | 14 ++++++++++----
.../physical/visitor/InsertLocalExchangeVisitor.java | 6 +++---
2 files changed, 13 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/3dfb4972/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
index caf21bc..4300c82 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
@@ -24,6 +24,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField;
import java.util.ArrayList;
@@ -36,6 +37,7 @@ public class HashPrelUtil {
public static final String HASH_EXPR_NAME = "E_X_P_R_H_A_S_H_F_I_E_L_D";
+ public static final int DIST_SEED = 1301011; // distribution seed
/**
* Interface for creating different forms of hash expression types.
* @param <T>
@@ -72,8 +74,9 @@ public class HashPrelUtil {
*/
public static <T> T createHashBasedPartitionExpression(
List<T> distFields,
+ T seed,
HashExpressionCreatorHelper<T> helper) {
- return createHashExpression(distFields, helper, true /*for distribution always hash as double*/);
+ return createHashExpression(distFields, seed, helper, true /*for distribution always hash as double*/);
}
/**
@@ -89,6 +92,7 @@ public class HashPrelUtil {
*/
public static <T> T createHashExpression(
List<T> inputExprs,
+ T seed,
HashExpressionCreatorHelper<T> helper,
boolean hashAsDouble) {
@@ -96,7 +100,7 @@ public class HashPrelUtil {
final String functionName = hashAsDouble ? HASH32_DOUBLE_FUNCTION_NAME : HASH32_FUNCTION_NAME;
- T func = helper.createCall(functionName, ImmutableList.of(inputExprs.get(0)));
+ T func = helper.createCall(functionName, ImmutableList.of(inputExprs.get(0), seed ));
for (int i = 1; i<inputExprs.size(); i++) {
func = helper.createCall(functionName, ImmutableList.of(inputExprs.get(i), func));
}
@@ -108,7 +112,8 @@ public class HashPrelUtil {
* Return a hash expression : hash32(field1, hash32(field2, hash32(field3, 0)));
*/
public static LogicalExpression getHashExpression(List<LogicalExpression> fields, boolean hashAsDouble){
- return createHashExpression(fields, HASH_HELPER_LOGICALEXPRESSION, hashAsDouble);
+ final LogicalExpression seed = ValueExpressions.getInt(0); // Hash Table seed
+ return createHashExpression(fields, seed, HASH_HELPER_LOGICALEXPRESSION, hashAsDouble);
}
@@ -134,6 +139,7 @@ public class HashPrelUtil {
expressions.add(new FieldReference(childFields.get(fields.get(i).getFieldId()), ExpressionPosition.UNKNOWN));
}
- return createHashBasedPartitionExpression(expressions, HASH_HELPER_LOGICALEXPRESSION);
+ final LogicalExpression distSeed = ValueExpressions.getInt(DIST_SEED);
+ return createHashBasedPartitionExpression(expressions, distSeed, HASH_HELPER_LOGICALEXPRESSION);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3dfb4972/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
index 6ebdba4..fba4cbe 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
@@ -19,14 +19,12 @@ package org.apache.drill.exec.planner.physical.visitor;
import com.google.common.collect.Lists;
-import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.exec.planner.physical.ExchangePrel;
import org.apache.drill.exec.planner.physical.HashPrelUtil;
import org.apache.drill.exec.planner.physical.HashPrelUtil.HashExpressionCreatorHelper;
import org.apache.drill.exec.planner.physical.HashToRandomExchangePrel;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.planner.physical.Prel;
-import org.apache.drill.exec.planner.physical.PrelUtil;
import org.apache.drill.exec.planner.physical.ProjectPrel;
import org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField;
import org.apache.drill.exec.planner.physical.UnorderedDeMuxExchangePrel;
@@ -40,6 +38,7 @@ import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
+import java.math.BigDecimal;
import java.util.Collections;
import java.util.List;
@@ -119,7 +118,8 @@ public class InsertLocalExchangeVisitor extends BasePrelVisitor<Prel, Void, Runt
}
outputFieldNames.add(HashPrelUtil.HASH_EXPR_NAME);
- updatedExpr.add(HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, hashHelper));
+ final RexNode distSeed = rexBuilder.makeBigintLiteral(BigDecimal.valueOf(HashPrelUtil.DIST_SEED)); // distribution seed
+ updatedExpr.add(HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, distSeed, hashHelper));
RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(), updatedExpr, outputFieldNames);
[2/3] drill git commit: DRILL-5313: Fix compilation issue in C++
connector
Posted by jn...@apache.org.
DRILL-5313: Fix compilation issue in C++ connector
DRILL-5301 and DRILL-5167 have conflicting changes, which causes
the C++ connector to not compile: the static symbol for the search
escape string has been removed as the server might use a different one.
Fix the issue by using the current search escape string (injected from the
meta to the internal drill client when querying metadata).
close #769
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/f0057e7c
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/f0057e7c
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/f0057e7c
Branch: refs/heads/master
Commit: f0057e7c5e88a32408cbbdd3b475a8d192077a69
Parents: d96914f
Author: Laurent Goujon <la...@dremio.com>
Authored: Thu Mar 2 20:03:42 2017 -0800
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Thu Mar 2 23:12:53 2017 -0800
----------------------------------------------------------------------
.../client/src/clientlib/drillClientImpl.cpp | 29 ++++++++++++--------
.../client/src/clientlib/drillClientImpl.hpp | 8 +++---
.../native/client/src/clientlib/metadata.cpp | 8 +++---
3 files changed, 25 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/f0057e7c/contrib/native/client/src/clientlib/drillClientImpl.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClientImpl.cpp b/contrib/native/client/src/clientlib/drillClientImpl.cpp
index ce3ab63..d768bf3 100644
--- a/contrib/native/client/src/clientlib/drillClientImpl.cpp
+++ b/contrib/native/client/src/clientlib/drillClientImpl.cpp
@@ -661,16 +661,18 @@ DrillClientQueryResult* DrillClientImpl::ExecuteQuery(const PreparedStatement& p
return sendMsg(factory, ::exec::user::RUN_QUERY, query);
}
-static void updateLikeFilter(exec::user::LikeFilter& likeFilter, const std::string& pattern) {
+static void updateLikeFilter(exec::user::LikeFilter& likeFilter, const std::string& pattern,
+ const std::string& searchEscapeString) {
likeFilter.set_pattern(pattern);
- likeFilter.set_escape(meta::DrillMetadata::s_searchEscapeString);
+ likeFilter.set_escape(searchEscapeString);
}
DrillClientCatalogResult* DrillClientImpl::getCatalogs(const std::string& catalogPattern,
+ const std::string& searchEscapeString,
Metadata::pfnCatalogMetadataListener listener,
void* listenerCtx) {
exec::user::GetCatalogsReq query;
- updateLikeFilter(*query.mutable_catalog_name_filter(), catalogPattern);
+ updateLikeFilter(*query.mutable_catalog_name_filter(), catalogPattern, searchEscapeString);
boost::function<DrillClientCatalogResult*(int32_t)> factory = boost::bind(
boost::factory<DrillClientCatalogResult*>(),
@@ -683,11 +685,12 @@ DrillClientCatalogResult* DrillClientImpl::getCatalogs(const std::string& catalo
DrillClientSchemaResult* DrillClientImpl::getSchemas(const std::string& catalogPattern,
const std::string& schemaPattern,
+ const std::string& searchEscapeString,
Metadata::pfnSchemaMetadataListener listener,
void* listenerCtx) {
exec::user::GetSchemasReq query;
- updateLikeFilter(*query.mutable_catalog_name_filter(), catalogPattern);
- updateLikeFilter(*query.mutable_schema_name_filter(), schemaPattern);
+ updateLikeFilter(*query.mutable_catalog_name_filter(), catalogPattern, searchEscapeString);
+ updateLikeFilter(*query.mutable_schema_name_filter(), schemaPattern, searchEscapeString);
boost::function<DrillClientSchemaResult*(int32_t)> factory = boost::bind(
boost::factory<DrillClientSchemaResult*>(),
@@ -702,12 +705,13 @@ DrillClientTableResult* DrillClientImpl::getTables(const std::string& catalogPat
const std::string& schemaPattern,
const std::string& tablePattern,
const std::vector<std::string>* tableTypes,
+ const std::string& searchEscapeString,
Metadata::pfnTableMetadataListener listener,
void* listenerCtx) {
exec::user::GetTablesReq query;
- updateLikeFilter(*query.mutable_catalog_name_filter(), catalogPattern);
- updateLikeFilter(*query.mutable_schema_name_filter(), schemaPattern);
- updateLikeFilter(*query.mutable_table_name_filter(), tablePattern);
+ updateLikeFilter(*query.mutable_catalog_name_filter(), catalogPattern, searchEscapeString);
+ updateLikeFilter(*query.mutable_schema_name_filter(), schemaPattern, searchEscapeString);
+ updateLikeFilter(*query.mutable_table_name_filter(), tablePattern, searchEscapeString);
if (tableTypes) {
std::copy(tableTypes->begin(), tableTypes->end(),
@@ -727,13 +731,14 @@ DrillClientColumnResult* DrillClientImpl::getColumns(const std::string& catalogP
const std::string& schemaPattern,
const std::string& tablePattern,
const std::string& columnsPattern,
+ const std::string& searchEscapeString,
Metadata::pfnColumnMetadataListener listener,
void* listenerCtx) {
exec::user::GetColumnsReq query;
- updateLikeFilter(*query.mutable_catalog_name_filter(), catalogPattern);
- updateLikeFilter(*query.mutable_schema_name_filter(), schemaPattern);
- updateLikeFilter(*query.mutable_table_name_filter(), tablePattern);
- updateLikeFilter(*query.mutable_column_name_filter(), columnsPattern);
+ updateLikeFilter(*query.mutable_catalog_name_filter(), catalogPattern, searchEscapeString);
+ updateLikeFilter(*query.mutable_schema_name_filter(), schemaPattern, searchEscapeString);
+ updateLikeFilter(*query.mutable_table_name_filter(), tablePattern, searchEscapeString);
+ updateLikeFilter(*query.mutable_column_name_filter(), columnsPattern, searchEscapeString);
boost::function<DrillClientColumnResult*(int32_t)> factory = boost::bind(
boost::factory<DrillClientColumnResult*>(),
http://git-wip-us.apache.org/repos/asf/drill/blob/f0057e7c/contrib/native/client/src/clientlib/drillClientImpl.hpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClientImpl.hpp b/contrib/native/client/src/clientlib/drillClientImpl.hpp
index bc6503d..d37076e 100644
--- a/contrib/native/client/src/clientlib/drillClientImpl.hpp
+++ b/contrib/native/client/src/clientlib/drillClientImpl.hpp
@@ -527,10 +527,10 @@ class DrillClientImpl : public DrillClientImplBase{
Handle* sendMsg(boost::function<Handle*(int32_t)> handleFactory, ::exec::user::RpcType type, const ::google::protobuf::Message& msg);
// metadata requests
- DrillClientCatalogResult* getCatalogs(const std::string& catalogPattern, Metadata::pfnCatalogMetadataListener listener, void* listenerCtx);
- DrillClientSchemaResult* getSchemas(const std::string& catalogPattern, const std::string& schemaPattern, Metadata::pfnSchemaMetadataListener listener, void* listenerCtx);
- DrillClientTableResult* getTables(const std::string& catalogPattern, const std::string& schemaPattern, const std::string& tablePattern, const std::vector<std::string>* tableTypes, Metadata::pfnTableMetadataListener listener, void* listenerCtx);
- DrillClientColumnResult* getColumns(const std::string& catalogPattern, const std::string& schemaPattern, const std::string& tablePattern, const std::string& columnPattern, Metadata::pfnColumnMetadataListener listener, void* listenerCtx);
+ DrillClientCatalogResult* getCatalogs(const std::string& catalogPattern, const std::string& searchEscapeString, Metadata::pfnCatalogMetadataListener listener, void* listenerCtx);
+ DrillClientSchemaResult* getSchemas(const std::string& catalogPattern, const std::string& schemaPattern, const std::string& searchEscapeString, Metadata::pfnSchemaMetadataListener listener, void* listenerCtx);
+ DrillClientTableResult* getTables(const std::string& catalogPattern, const std::string& schemaPattern, const std::string& tablePattern, const std::vector<std::string>* tableTypes, const std::string& searchEscapeString, Metadata::pfnTableMetadataListener listener, void* listenerCtx);
+ DrillClientColumnResult* getColumns(const std::string& catalogPattern, const std::string& schemaPattern, const std::string& tablePattern, const std::string& columnPattern, const std::string& searchEscapeString, Metadata::pfnColumnMetadataListener listener, void* listenerCtx);
// SASL exchange
connectionStatus_t handleAuthentication(const DrillUserProperties *userProperties);
http://git-wip-us.apache.org/repos/asf/drill/blob/f0057e7c/contrib/native/client/src/clientlib/metadata.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/metadata.cpp b/contrib/native/client/src/clientlib/metadata.cpp
index 7c2a7a8..ad08a40 100644
--- a/contrib/native/client/src/clientlib/metadata.cpp
+++ b/contrib/native/client/src/clientlib/metadata.cpp
@@ -1143,7 +1143,7 @@ uint32_t DrillMetadata::getServerPatchVersion() const {
}
status_t DrillMetadata::getCatalogs(const std::string& catalogPattern, Metadata::pfnCatalogMetadataListener listener, void* listenerCtx, QueryHandle_t* qHandle) {
- DrillClientCatalogResult* result = m_client.getCatalogs(catalogPattern, listener, listenerCtx);
+ DrillClientCatalogResult* result = m_client.getCatalogs(catalogPattern, m_searchEscapeString, listener, listenerCtx);
if(result==NULL){
*qHandle=NULL;
return static_cast<status_t>(m_client.getError()->status);
@@ -1152,7 +1152,7 @@ status_t DrillMetadata::getCatalogs(const std::string& catalogPattern, Metadata:
return QRY_SUCCESS;
}
status_t DrillMetadata::getSchemas(const std::string& catalogPattern, const std::string& schemaPattern, Metadata::pfnSchemaMetadataListener listener, void* listenerCtx, QueryHandle_t* qHandle) {
- DrillClientSchemaResult* result = m_client.getSchemas(catalogPattern, schemaPattern, listener, listenerCtx);
+ DrillClientSchemaResult* result = m_client.getSchemas(catalogPattern, schemaPattern, m_searchEscapeString, listener, listenerCtx);
if(result==NULL){
*qHandle=NULL;
return static_cast<status_t>(m_client.getError()->status);
@@ -1161,7 +1161,7 @@ status_t DrillMetadata::getSchemas(const std::string& catalogPattern, const std:
return QRY_SUCCESS;
}
status_t DrillMetadata::getTables(const std::string& catalogPattern, const std::string& schemaPattern, const std::string& tablePattern, const std::vector<std::string>* tableTypes, Metadata::pfnTableMetadataListener listener, void* listenerCtx, QueryHandle_t* qHandle) {
- DrillClientTableResult* result = m_client.getTables(catalogPattern, schemaPattern, tablePattern, tableTypes, listener, listenerCtx);
+ DrillClientTableResult* result = m_client.getTables(catalogPattern, schemaPattern, tablePattern, tableTypes, m_searchEscapeString, listener, listenerCtx);
if(result==NULL){
*qHandle=NULL;
return static_cast<status_t>(m_client.getError()->status);
@@ -1170,7 +1170,7 @@ status_t DrillMetadata::getTables(const std::string& catalogPattern, const std::
return QRY_SUCCESS;
}
status_t DrillMetadata::getColumns(const std::string& catalogPattern, const std::string& schemaPattern, const std:: string& tablePattern, const std::string& columnPattern, Metadata::pfnColumnMetadataListener listener, void* listenerCtx, QueryHandle_t* qHandle) {
- DrillClientColumnResult* result = m_client.getColumns(catalogPattern, schemaPattern, tablePattern, columnPattern, listener, listenerCtx);
+ DrillClientColumnResult* result = m_client.getColumns(catalogPattern, schemaPattern, tablePattern, columnPattern, m_searchEscapeString, listener, listenerCtx);
if(result==NULL){
*qHandle=NULL;
return static_cast<status_t>(m_client.getError()->status);