You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/05/13 19:36:45 UTC
[1/7] git commit: DRILL-607, DRILL-593: Fix casting from Int,
BigInt to Decimal28, Decimal38
Repository: incubator-drill
Updated Branches:
refs/heads/master 63b034676 -> e7a486d78
DRILL-607, DRILL-593: Fix casting from Int, BigInt to Decimal28, Decimal38
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2631bbce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2631bbce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2631bbce
Branch: refs/heads/master
Commit: 2631bbce6dcb1e5120462e3496ee9c4ae4caa71e
Parents: 63b0346
Author: mehant <me...@gmail.com>
Authored: Wed May 7 01:45:04 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Mon May 12 21:33:14 2014 -0700
----------------------------------------------------------------------
.../codegen/templates/Decimal/CastIntDecimal.java | 11 ++---------
.../apache/drill/jdbc/test/TestFunctionsQuery.java | 14 ++++++++++++++
2 files changed, 16 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2631bbce/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
index 1efe21f..525566a 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
@@ -59,7 +59,6 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
}
public void eval() {
-
out.scale = (int) scale.value;
out.precision = (int) precision.value;
@@ -81,20 +80,14 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// check if input is a negative number and store the sign
if (in.value < 0) {
out.sign = true;
- /* we are going to split the input into base 1 billion numbers
- * by dividing, if we leave the input to be negative all the splits
- * will be negative. We store the sign in the output at the end in the
- * most significant bit
- */
- in.value = in.value * -1;
}
// Figure out how many array positions to be left for the scale part
int scaleSize = org.apache.drill.common.util.DecimalUtility.roundUp((int) scale.value);
int integerIndex = (${type.arraySize} - scaleSize - 1);
- while (in.value > 0 && integerIndex >= 0) {
- out.setInteger(integerIndex, (int) (in.value % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE));
+ while (in.value != 0 && integerIndex >= 0) {
+ out.setInteger(integerIndex--, (int) Math.abs((in.value % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE)));
in.value = in.value / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2631bbce/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
index 1c78b5f..79f29c8 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.jdbc.test;
+import java.lang.Exception;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -416,4 +417,17 @@ public class TestFunctionsQuery {
);
}
+ @Test
+ public void testIntMinToDecimal() throws Exception {
+ String query = "select cast((employee_id - employee_id + -2147483648) as decimal(28, 2)) as DEC_28," +
+ "cast((employee_id - employee_id + -2147483648) as decimal(18, 2)) as DEC_18 from " +
+ "cp.`employee.json` limit 1";
+
+ JdbcAssert.withNoDefaultSchema()
+ .sql(query)
+ .returns(
+ "DEC_28=-2147483648.00; " +
+ "DEC_18=-2147483648.00\n"
+ );
+ }
}
[4/7] git commit: Remove the *.orig files lying in the repo.
Posted by ja...@apache.org.
Remove the *.orig files lying in the repo.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c5f5e282
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c5f5e282
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c5f5e282
Branch: refs/heads/master
Commit: c5f5e282b20838d75f58f42f27403b18f15f9a22
Parents: 26fee8a
Author: vkorukanti <ve...@gmail.com>
Authored: Mon May 12 12:29:32 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Mon May 12 21:33:31 2014 -0700
----------------------------------------------------------------------
.../exec/expr/fn/impl/HashFunctions.java.orig | 252 -------------------
.../drill/exec/ops/QueryContext.java.orig | 135 ----------
.../apache/drill/exec/server/Drillbit.java.orig | 141 -----------
.../drill/exec/server/DrillbitContext.java.orig | 157 ------------
.../exec/store/StoragePluginRegistry.java.orig | 243 ------------------
5 files changed, 928 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c5f5e282/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java.orig
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java.orig b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java.orig
deleted file mode 100644
index 7eeb730..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java.orig
+++ /dev/null
@@ -1,252 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.fn.impl;
-
-import org.apache.drill.exec.expr.DrillSimpleFunc;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
-import org.apache.drill.exec.expr.annotations.Output;
-import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.holders.*;
-import org.apache.drill.exec.record.RecordBatch;
-
-public class HashFunctions {
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
- public static class NullableFloatHash implements DrillSimpleFunc {
-
- @Param NullableFloat4Holder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- if (in.isSet == 0)
- out.value = 0;
- else
- out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(Float.floatToIntBits(in.value)).asInt();
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
- public static class FloatHash implements DrillSimpleFunc {
-
- @Param Float4Holder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(Float.floatToIntBits(in.value)).asInt();
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
- public static class NullableDoubleHash implements DrillSimpleFunc {
-
- @Param NullableFloat8Holder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- if (in.isSet == 0)
- out.value = 0;
- else
- out.value = com.google.common.hash.Hashing.murmur3_128().hashLong(Double.doubleToLongBits(in.value)).asInt();
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
- public static class DoubleHash implements DrillSimpleFunc {
-
- @Param Float8Holder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- out.value = com.google.common.hash.Hashing.murmur3_128().hashLong(Double.doubleToLongBits(in.value)).asInt();
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
- public static class NullableVarBinaryHash implements DrillSimpleFunc {
-
- @Param NullableVarBinaryHolder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- if (in.isSet == 0)
- out.value = 0;
- else
- out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
- public static class NullableVarCharHash implements DrillSimpleFunc {
-
- @Param NullableVarCharHolder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- if (in.isSet == 0)
- out.value = 0;
- else
- out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
- public static class NullableVar16CharHash implements DrillSimpleFunc {
-
- @Param NullableVar16CharHolder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- if (in.isSet == 0)
- out.value = 0;
- else
- out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
- public static class NullableBigIntHash implements DrillSimpleFunc {
-
- @Param NullableBigIntHolder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- // TODO: implement hash function for other types
- if (in.isSet == 0)
- out.value = 0;
- else
- out.value = com.google.common.hash.Hashing.murmur3_128().hashLong(in.value).asInt();
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
- public static class NullableIntHash implements DrillSimpleFunc {
- @Param NullableIntHolder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- // TODO: implement hash function for other types
- if (in.isSet == 0)
- out.value = 0;
- else
- out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(in.value).asInt();
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
- public static class VarBinaryHash implements DrillSimpleFunc {
-
- @Param VarBinaryHolder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
- public static class VarCharHash implements DrillSimpleFunc {
-
- @Param VarCharHolder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
- }
- }
-
-<<<<<<< HEAD
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
- public static class Var16CharHash implements DrillSimpleFunc {
-
- @Param Var16CharHolder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
- }
- }
-
-=======
->>>>>>> 450e9e0... Support Complex Types
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
- public static class HashBigInt implements DrillSimpleFunc {
-
- @Param BigIntHolder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- // TODO: implement hash function for other types
- out.value = com.google.common.hash.Hashing.murmur3_128().hashLong(in.value).asInt();
- }
- }
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
- public static class IntHash implements DrillSimpleFunc {
- @Param IntHolder in;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- public void eval() {
- // TODO: implement hash function for other types
- out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(in.value).asInt();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c5f5e282/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java.orig
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java.orig b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java.orig
deleted file mode 100644
index 4f050a6..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java.orig
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ops;
-
-import java.util.Collection;
-
-import net.hydromatic.optiq.SchemaPlus;
-import net.hydromatic.optiq.tools.Frameworks;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.exceptions.ExpressionParsingException;
-import org.apache.drill.exec.cache.DistributedCache;
-import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.planner.physical.PlannerSettings;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.proto.UserBitShared.QueryId;
-import org.apache.drill.exec.rpc.control.WorkEventBus;
-import org.apache.drill.exec.rpc.data.DataConnectionCreator;
-import org.apache.drill.exec.rpc.user.UserSession;
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.StoragePluginRegistry;
-
-public class QueryContext{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryContext.class);
-
- private final QueryId queryId;
- private final DrillbitContext drillbitContext;
- private final WorkEventBus workBus;
- private UserSession session;
- public final Multitimer<QuerySetup> timer;
- private final PlannerSettings plannerSettings;
-
- public QueryContext(UserSession session, QueryId queryId, DrillbitContext drllbitContext) {
- super();
- this.queryId = queryId;
- this.drillbitContext = drllbitContext;
- this.workBus = drllbitContext.getWorkBus();
- this.session = session;
- this.timer = new Multitimer<>(QuerySetup.class);
- this.plannerSettings = new PlannerSettings();
- }
-
- public PlannerSettings getPlannerSettings(){
- return plannerSettings;
- }
-
- public UserSession getSession(){
- return session;
- }
-
- public SchemaPlus getNewDefaultSchema(){
- SchemaPlus rootSchema = getRootSchema();
- SchemaPlus defaultSchema = session.getDefaultSchema(rootSchema);
- if(defaultSchema == null){
- return rootSchema;
- }else{
- return defaultSchema;
- }
- }
-
- public SchemaPlus getRootSchema(){
- SchemaPlus rootSchema = Frameworks.createRootSchema();
- drillbitContext.getSchemaFactory().registerSchemas(session.getUser(), rootSchema);
- return rootSchema;
- }
-
-<<<<<<< HEAD
-=======
- public Object getOptionValue(String name){
- return drillbitContext.getGlobalDrillOptions().getOption(name);
- }
-
- public void setOptionValue(String name, String value) throws ExpressionParsingException {
- drillbitContext.getGlobalDrillOptions().setOption(name, value);
- }
-
->>>>>>> Drill 381 - implementation of session and global options.
- public DrillbitEndpoint getCurrentEndpoint(){
- return drillbitContext.getEndpoint();
- }
-
- public QueryId getQueryId() {
- return queryId;
- }
-
- public StoragePluginRegistry getStorage(){
- return drillbitContext.getStorage();
- }
-
-
- public DistributedCache getCache(){
- return drillbitContext.getCache();
- }
-
- public Collection<DrillbitEndpoint> getActiveEndpoints(){
- return drillbitContext.getBits();
- }
-
- public PhysicalPlanReader getPlanReader(){
- return drillbitContext.getPlanReader();
- }
-
- public DataConnectionCreator getDataConnectionsPool(){
- return drillbitContext.getDataConnectionsPool();
- }
-
- public DrillConfig getConfig(){
- return drillbitContext.getConfig();
- }
-
- public WorkEventBus getWorkBus(){
- return workBus;
- }
-
- public FunctionImplementationRegistry getFunctionRegistry(){
- return drillbitContext.getFunctionImplementationRegistry();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c5f5e282/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java.orig
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java.orig b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java.orig
deleted file mode 100644
index 5e72717..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java.orig
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.server;
-
-import java.io.Closeable;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.cache.DistributedCache;
-import org.apache.drill.exec.cache.DistributedMultiMap;
-import org.apache.drill.exec.cache.HazelCache;
-import org.apache.drill.exec.coord.ClusterCoordinator;
-import org.apache.drill.exec.coord.ClusterCoordinator.RegistrationHandle;
-import org.apache.drill.exec.coord.ZKClusterCoordinator;
-import org.apache.drill.exec.exception.DrillbitStartupException;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.service.ServiceEngine;
-import org.apache.drill.exec.work.WorkManager;
-
-import com.google.common.io.Closeables;
-
-/**
- * Starts, tracks and stops all the required services for a Drillbit daemon to work.
- */
-public class Drillbit implements Closeable{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Drillbit.class);
-
- public static Drillbit start(StartupOptions options) throws DrillbitStartupException {
- return start(DrillConfig.create(options.getConfigLocation()));
- }
-
- public static Drillbit start(DrillConfig config) throws DrillbitStartupException {
- Drillbit bit;
- try {
- logger.debug("Setting up Drillbit.");
- bit = new Drillbit(config, null);
- } catch (Exception ex) {
- throw new DrillbitStartupException("Failure while initializing values in Drillbit.", ex);
- }
- try {
- logger.debug("Starting Drillbit.");
- bit.run();
- } catch (Exception e) {
- throw new DrillbitStartupException("Failure during initial startup of Drillbit.", e);
- }
- return bit;
- }
-
- public static void main(String[] cli) throws DrillbitStartupException {
- StartupOptions options = StartupOptions.parse(cli);
- start(options);
- }
-
- final ClusterCoordinator coord;
- final ServiceEngine engine;
- final DistributedCache cache;
- final WorkManager manager;
- final BootStrapContext context;
-
- private volatile RegistrationHandle handle;
-
- public Drillbit(DrillConfig config, RemoteServiceSet serviceSet) throws Exception {
- if(serviceSet != null){
- this.context = new BootStrapContext(config);
- this.manager = new WorkManager(context);
- this.coord = serviceSet.getCoordinator();
- this.engine = new ServiceEngine(manager.getControlMessageHandler(), manager.getUserWorker(), context, manager.getWorkBus(), manager.getDataHandler());
- this.cache = serviceSet.getCache();
- }else{
- Runtime.getRuntime().addShutdownHook(new ShutdownThread(config));
- this.context = new BootStrapContext(config);
- this.manager = new WorkManager(context);
- this.coord = new ZKClusterCoordinator(config);
- this.engine = new ServiceEngine(manager.getControlMessageHandler(), manager.getUserWorker(), context, manager.getWorkBus(), manager.getDataHandler());
- this.cache = new HazelCache(config, context.getAllocator());
- }
- }
-
- public void run() throws Exception {
- coord.start(10000);
- DrillbitEndpoint md = engine.start();
- cache.run();
-<<<<<<< HEAD
- manager.getContext().getStorage().init();
-=======
- manager.start(md, cache, engine.getController(), engine.getDataConnectionCreator(), coord);
->>>>>>> Drill 381 - implementation of session and global options.
- handle = coord.register(md);
- }
-
- public void close() {
- if (coord != null) coord.unregister(handle);
-
- try {
- Thread.sleep(context.getConfig().getInt(ExecConstants.ZK_REFRESH) * 2);
- } catch (InterruptedException e) {
- logger.warn("Interrupted while sleeping during coordination deregistration.");
- }
-
- Closeables.closeQuietly(engine);
- Closeables.closeQuietly(coord);
- Closeables.closeQuietly(manager);
- Closeables.closeQuietly(context);
- logger.info("Shutdown completed.");
- }
-
- private class ShutdownThread extends Thread {
- ShutdownThread(DrillConfig config) {
- this.setName("ShutdownHook");
- }
-
- @Override
- public void run() {
- logger.info("Received shutdown request.");
- close();
- }
-
- }
- public ClusterCoordinator getCoordinator(){
- return coord;
- }
-
- public DrillbitContext getContext(){
- return this.manager.getContext();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c5f5e282/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java.orig
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java.orig b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java.orig
deleted file mode 100644
index cdfcf60..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java.orig
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.server;
-
-import io.netty.channel.nio.NioEventLoopGroup;
-
-import java.util.Collection;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.cache.DistributedCache;
-import org.apache.drill.exec.cache.DistributedMultiMap;
-import org.apache.drill.exec.coord.ClusterCoordinator;
-import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
-import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.rpc.control.Controller;
-import org.apache.drill.exec.rpc.control.WorkEventBus;
-import org.apache.drill.exec.rpc.data.DataConnectionCreator;
-import org.apache.drill.exec.store.StoragePluginRegistry;
-import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory;
-import org.apache.drill.exec.store.StoragePlugin;
-
-import com.codahale.metrics.MetricRegistry;
-import com.google.common.base.Preconditions;
-
-public class DrillbitContext {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillbitContext.class);
-
- private final BootStrapContext context;
-
- private PhysicalPlanReader reader;
- private final ClusterCoordinator coord;
- private final DataConnectionCreator connectionsPool;
- private final DistributedCache cache;
- private final DrillbitEndpoint endpoint;
- private final StoragePluginRegistry storagePlugins;
- private final OperatorCreatorRegistry operatorCreatorRegistry;
- private final Controller controller;
- private final WorkEventBus workBus;
- private final FunctionImplementationRegistry functionRegistry;
-<<<<<<< HEAD
-
-=======
- private final DistributedGlobalOptions globalDrillOptions;
-
->>>>>>> Drill 381 - implementation of session and global options.
- public DrillbitContext(DrillbitEndpoint endpoint, BootStrapContext context, ClusterCoordinator coord, Controller controller, DataConnectionCreator connectionsPool, DistributedCache cache, WorkEventBus workBus) {
- super();
- Preconditions.checkNotNull(endpoint);
- Preconditions.checkNotNull(context);
- Preconditions.checkNotNull(controller);
- Preconditions.checkNotNull(connectionsPool);
- this.workBus = workBus;
- this.controller = controller;
- this.context = context;
- this.coord = coord;
- this.connectionsPool = connectionsPool;
- this.cache = cache;
- this.endpoint = endpoint;
- this.storagePlugins = new StoragePluginRegistry(this);
- this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(), endpoint, storagePlugins);
- this.operatorCreatorRegistry = new OperatorCreatorRegistry(context.getConfig());
- this.functionRegistry = new FunctionImplementationRegistry(context.getConfig());
- this.globalDrillOptions = new DistributedGlobalOptions(this.cache);
- }
-
-<<<<<<< HEAD
-=======
-
->>>>>>> Drill 381 - implementation of session and global options.
- public FunctionImplementationRegistry getFunctionImplementationRegistry() {
- return functionRegistry;
- }
-
- public WorkEventBus getWorkBus(){
- return workBus;
- }
-<<<<<<< HEAD
-=======
-
- public DistributedGlobalOptions getGlobalDrillOptions() {
- return globalDrillOptions;
- }
->>>>>>> Drill 381 - implementation of session and global options.
-
- public DrillbitEndpoint getEndpoint(){
- return endpoint;
- }
-
- public DrillConfig getConfig() {
- return context.getConfig();
- }
-
- public Collection<DrillbitEndpoint> getBits(){
- return coord.getAvailableEndpoints();
- }
-
- public BufferAllocator getAllocator(){
- return context.getAllocator();
- }
-
- public OperatorCreatorRegistry getOperatorCreatorRegistry() {
- return operatorCreatorRegistry;
- }
-
- public StoragePluginRegistry getStorage(){
- return this.storagePlugins;
- }
-
- public NioEventLoopGroup getBitLoopGroup(){
- return context.getBitLoopGroup();
- }
-
-
- public DataConnectionCreator getDataConnectionsPool(){
- return connectionsPool;
- }
-
- public Controller getController(){
- return controller;
- }
-
- public MetricRegistry getMetrics(){
- return context.getMetrics();
- }
-
- public DistributedCache getCache(){
- return cache;
- }
-
- public PhysicalPlanReader getPlanReader(){
- return reader;
- }
-
- public DrillSchemaFactory getSchemaFactory(){
- return storagePlugins.getSchemaFactory();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c5f5e282/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java.orig
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java.orig b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java.orig
deleted file mode 100644
index 2c75651..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java.orig
+++ /dev/null
@@ -1,243 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.net.URL;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import com.google.common.base.Preconditions;
-import net.hydromatic.linq4j.expressions.DefaultExpression;
-import net.hydromatic.linq4j.expressions.Expression;
-import net.hydromatic.optiq.SchemaPlus;
-import net.hydromatic.optiq.tools.RuleSet;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.logical.FormatPluginConfig;
-import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.common.util.PathScanner;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.planner.logical.DrillRuleSets;
-import org.apache.drill.exec.cache.DistributedMap;
-import org.apache.drill.exec.cache.JacksonDrillSerializable.StoragePluginsSerializable;
-import org.apache.drill.exec.exception.DrillbitStartupException;
-import org.apache.drill.exec.planner.logical.StoragePlugins;
-import org.apache.drill.exec.rpc.user.DrillUser;
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.dfs.FileSystemPlugin;
-import org.apache.drill.exec.store.dfs.FormatPlugin;
-import org.apache.drill.exec.store.ischema.InfoSchemaConfig;
-import org.apache.drill.exec.store.ischema.InfoSchemaStoragePlugin;
-import org.eigenbase.relopt.RelOptRule;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSet.Builder;
-import com.google.common.io.Resources;
-import org.apache.drill.exec.store.options.OptionValueStorageConfig;
-import org.apache.drill.exec.store.options.OptionValueStoragePlugin;
-
-
-public class StoragePluginRegistry implements Iterable<Map.Entry<String, StoragePlugin>>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginRegistry.class);
-
- private Map<Object, Constructor<? extends StoragePlugin>> availablePlugins = new HashMap<Object, Constructor<? extends StoragePlugin>>();
- private ImmutableMap<String, StoragePlugin> plugins;
-
- private DrillbitContext context;
- private final DrillSchemaFactory schemaFactory = new DrillSchemaFactory();
-
- private RuleSet storagePluginsRuleSet;
-
- private static final Expression EXPRESSION = new DefaultExpression(Object.class);
-
- public StoragePluginRegistry(DrillbitContext context) {
- try{
- this.context = context;
- }catch(RuntimeException e){
- logger.error("Failure while loading storage plugin registry.", e);
- throw new RuntimeException("Faiure while reading and loading storage plugin configuration.", e);
- }
- }
-
- @SuppressWarnings("unchecked")
- public void init() throws DrillbitStartupException {
- DrillConfig config = context.getConfig();
- Collection<Class<? extends StoragePlugin>> plugins = PathScanner.scanForImplementations(StoragePlugin.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES));
- logger.debug("Loading storage plugins {}", plugins);
- for(Class<? extends StoragePlugin> plugin: plugins){
- int i =0;
- for(Constructor<?> c : plugin.getConstructors()){
- Class<?>[] params = c.getParameterTypes();
- if(params.length != 3
- || params[1] != DrillbitContext.class
- || !StoragePluginConfig.class.isAssignableFrom(params[0])
- || params[2] != String.class){
- logger.info("Skipping StoragePlugin constructor {} for plugin class {} since it doesn't implement a [constructor(StoragePluginConfig, DrillbitContext, String)]", c, plugin);
- continue;
- }
- availablePlugins.put(params[0], (Constructor<? extends StoragePlugin>) c);
- i++;
- }
- if(i == 0){
- logger.debug("Skipping registration of StoragePlugin {} as it doesn't have a constructor with the parameters of (StorangePluginConfig, Config)", plugin.getCanonicalName());
- }
- }
-
- // create registered plugins defined in "storage-plugins.json"
- this.plugins = ImmutableMap.copyOf(createPlugins());
-
- // query registered engines for optimizer rules and build the storage plugin RuleSet
- Builder<RelOptRule> setBuilder = ImmutableSet.builder();
- for (StoragePlugin plugin : this.plugins.values()) {
- Set<StoragePluginOptimizerRule> rules = plugin.getOptimizerRules();
- if (rules != null && rules.size() > 0) {
- setBuilder.addAll(rules);
- }
- }
- this.storagePluginsRuleSet = DrillRuleSets.create(setBuilder.build());
- }
-
- private Map<String, StoragePlugin> createPlugins() throws DrillbitStartupException {
- /*
- * Check if "storage-plugins.json" exists. Also check if "storage-plugins" object exists in Distributed Cache.
- * If both exist, check that they are the same. If they differ, throw exception. If "storage-plugins.json" exists, but
- * nothing found in cache, then add it to the cache. If neither are found, throw exception.
- */
- StoragePlugins plugins = null;
- StoragePlugins cachedPlugins = null;
- Map<String, StoragePlugin> activePlugins = new HashMap<String, StoragePlugin>();
- try{
- URL url = Resources.class.getClassLoader().getResource("storage-plugins.json");
- if (url != null) {
- String pluginsData = Resources.toString(url, Charsets.UTF_8);
- plugins = context.getConfig().getMapper().readValue(pluginsData, StoragePlugins.class);
- }
- DistributedMap<StoragePluginsSerializable> map = context.getCache().getMap(StoragePluginsSerializable.class);
- StoragePluginsSerializable cachedPluginsSerializable = map.get("storage-plugins");
- if (cachedPluginsSerializable != null) {
- cachedPlugins = cachedPluginsSerializable.getObj();
- logger.debug("Found cached storage plugin config: {}", cachedPlugins);
- } else {
- Preconditions.checkNotNull(plugins,"No storage plugin configuration found");
- logger.debug("caching storage plugin config {}", plugins);
- map.put("storage-plugins", new StoragePluginsSerializable(context, plugins));
- cachedPluginsSerializable = map.get("storage-plugins");
- cachedPlugins = cachedPluginsSerializable.getObj();
- }
- if(!(plugins == null || cachedPlugins.equals(plugins))) {
- logger.error("Storage plugin config mismatch. {}. {}", plugins, cachedPlugins);
- throw new DrillbitStartupException("Storage plugin config mismatch");
- }
- logger.debug("using plugin config: {}", cachedPlugins);
- }catch(IOException e){
- throw new IllegalStateException("Failure while reading storage plugins data.", e);
- }
-
- for(Map.Entry<String, StoragePluginConfig> config : cachedPlugins){
- try{
- StoragePlugin plugin = create(config.getKey(), config.getValue());
- activePlugins.put(config.getKey(), plugin);
- }catch(ExecutionSetupException e){
- logger.error("Failure while setting up StoragePlugin with name: '{}'.", config.getKey(), e);
- }
- }
-<<<<<<< HEAD
- activePlugins.put("INFORMATION_SCHEMA", new InfoSchemaStoragePlugin(new InfoSchemaConfig(), context, "INFORMATION_SCHEMA"));
-
- return activePlugins;
-=======
- activeEngines.put("INFORMATION_SCHEMA", new InfoSchemaStoragePlugin(new InfoSchemaConfig(), context, "INFORMATION_SCHEMA"));
- activeEngines.put("OPTIONS", new OptionValueStoragePlugin(new OptionValueStorageConfig(), context, "OPTIONS"));
-
- return activeEngines;
->>>>>>> Drill 381 - implementation of session and global options.
- }
-
- public StoragePlugin getPlugin(String registeredStoragePluginName) throws ExecutionSetupException {
- return plugins.get(registeredStoragePluginName);
- }
-
- public StoragePlugin getPlugin(StoragePluginConfig config) throws ExecutionSetupException {
- if(config instanceof NamedStoragePluginConfig){
- return plugins.get(((NamedStoragePluginConfig) config).name);
- }else{
- // TODO: for now, we'll throw away transient configs. we really ought to clean these up.
- return create(null, config);
- }
- }
-
- public FormatPlugin getFormatPlugin(StoragePluginConfig storageConfig, FormatPluginConfig formatConfig) throws ExecutionSetupException{
- StoragePlugin p = getPlugin(storageConfig);
- if(!(p instanceof FileSystemPlugin)) throw new ExecutionSetupException(String.format("You tried to request a format plugin for a storage plugin that wasn't of type FileSystemPlugin. The actual type of plugin was %s.", p.getClass().getName()));
- FileSystemPlugin storage = (FileSystemPlugin) p;
- return storage.getFormatPlugin(formatConfig);
- }
-
- private StoragePlugin create(String name, StoragePluginConfig pluginConfig) throws ExecutionSetupException {
- StoragePlugin plugin = null;
- Constructor<? extends StoragePlugin> c = availablePlugins.get(pluginConfig.getClass());
- if (c == null)
- throw new ExecutionSetupException(String.format("Failure finding StoragePlugin constructor for config %s",
- pluginConfig));
- try {
- plugin = c.newInstance(pluginConfig, context, name);
- return plugin;
- } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
- Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException) e).getTargetException() : e;
- if (t instanceof ExecutionSetupException)
- throw ((ExecutionSetupException) t);
- throw new ExecutionSetupException(String.format(
- "Failure setting up new storage plugin configuration for config %s", pluginConfig), t);
- }
- }
-
- @Override
- public Iterator<Entry<String, StoragePlugin>> iterator() {
- return plugins.entrySet().iterator();
- }
-
- public RuleSet getStoragePluginRuleSet() {
- return storagePluginsRuleSet;
- }
-
- public DrillSchemaFactory getSchemaFactory(){
- return schemaFactory;
- }
-
- public class DrillSchemaFactory implements SchemaFactory{
-
- @Override
- public void registerSchemas(DrillUser user, SchemaPlus parent) {
- for(Map.Entry<String, StoragePlugin> e : plugins.entrySet()){
- e.getValue().registerSchemas(user, parent);
- }
- }
-
- }
-
-}
[3/7] git commit: DRILL-558: Remove workaround in OptiqScanner for
skipping "metadata" subschema.
Posted by ja...@apache.org.
DRILL-558: Remove workaround in OptiqScanner for skipping "metadata" subschema.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/26fee8a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/26fee8a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/26fee8a2
Branch: refs/heads/master
Commit: 26fee8a2a2050c49de98ea20d4d05be27d840892
Parents: b75782d
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Apr 22 20:47:43 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Mon May 12 21:33:26 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/drill/exec/ops/FragmentContext.java | 2 +-
.../src/main/java/org/apache/drill/exec/ops/QueryContext.java | 2 +-
.../org/apache/drill/exec/store/ischema/OptiqProvider.java | 7 ++-----
.../src/test/java/org/apache/drill/PlanningBase.java | 2 +-
.../java/org/apache/drill/exec/store/TestOrphanSchema.java | 2 +-
.../org/apache/drill/exec/store/ischema/OrphanSchema.java | 2 +-
6 files changed, 7 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/26fee8a2/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 38d76e0..3f9f181 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -128,7 +128,7 @@ public class FragmentContext implements Closeable {
"This is a non-root fragment."));
return null;
} else {
- SchemaPlus root = Frameworks.createRootSchema();
+ SchemaPlus root = Frameworks.createRootSchema(false);
context.getStorage().getSchemaFactory().registerSchemas(connection.getSession(), root);
return root;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/26fee8a2/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index e541200..36def17 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -79,7 +79,7 @@ public class QueryContext{
}
public SchemaPlus getRootSchema(){
- SchemaPlus rootSchema = Frameworks.createRootSchema();
+ SchemaPlus rootSchema = Frameworks.createRootSchema(false);
drillbitContext.getSchemaFactory().registerSchemas(session, rootSchema);
return rootSchema;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/26fee8a2/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
index 3ae1138..cb4f3d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
@@ -59,8 +59,7 @@ public class OptiqProvider {
static public class Schemata extends Abstract {
@Override
public boolean visitSchema(String schemaName, Schema schema) {
- if (schemaName != null && schemaName != "" &&
- !schemaName.equals("metadata") /* workaround until Drill upgrades to Optiq version which has the fix for #256 */) {
+ if (schemaName != null && schemaName != "") {
writeRow("DRILL", schemaName, "<owner>");
}
return false;
@@ -174,9 +173,7 @@ public class OptiqProvider {
* If the schema visitor returns true, then visit the tables.
* If the table visitor returns true, then visit the fields (columns).
*/
- public boolean visitSchema(String schemaName, Schema schema){
- return !schemaName.equals("metadata"); //workaround until Drill upgrades to Optiq version which has the fix for #256
- }
+ public boolean visitSchema(String schemaName, Schema schema){return true;}
public boolean visitTableName(String schemaName, String tableName){return true;}
public boolean visitTable(String schemaName, String tableName, Table table){return true;}
public boolean visitField(String schemaName, String tableName, RelDataTypeField field){return true;}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/26fee8a2/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
index e030da5..99e712b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
@@ -93,7 +93,7 @@ public class PlanningBase extends ExecTest{
final StoragePluginRegistry registry = new StoragePluginRegistry(dbContext);
registry.init();
final FunctionImplementationRegistry functionRegistry = new FunctionImplementationRegistry(config);
- final SchemaPlus root = Frameworks.createRootSchema();
+ final SchemaPlus root = Frameworks.createRootSchema(false);
registry.getSchemaFactory().registerSchemas(new UserSession(null, null, null), root);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/26fee8a2/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
index 864793a..63bc0a9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
@@ -55,7 +55,7 @@ public class TestOrphanSchema extends ExecTest {
bitContext.getCache().run();
StoragePluginRegistry r = new StoragePluginRegistry(bitContext);
- SchemaPlus plus = Frameworks.createRootSchema();
+ SchemaPlus plus = Frameworks.createRootSchema(false);
r.init();
r.getSchemaFactory().registerSchemas(new UserSession(null, null, null), plus);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/26fee8a2/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
index b4ed88a..acb5929 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
@@ -55,7 +55,7 @@ public class OrphanSchema {
// Using the mock context, get the orphan schema.
StoragePluginRegistry r = new StoragePluginRegistry(bitContext);
r.init();
- SchemaPlus plus = Frameworks.createRootSchema();
+ SchemaPlus plus = Frameworks.createRootSchema(false);
r.getSchemaFactory().registerSchemas(new UserSession(null, null, null), plus);
return plus;
}
[5/7] git commit: DRILL-623: Unable to run query containing quoted
schema name
Posted by ja...@apache.org.
DRILL-623: Unable to run query containing quoted schema name
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/d414d5dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/d414d5dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/d414d5dc
Branch: refs/heads/master
Commit: d414d5dccbda7b901260d66448367678bdced235
Parents: c5f5e28
Author: vkorukanti <ve...@gmail.com>
Authored: Mon May 12 17:33:41 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Mon May 12 21:33:34 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/store/AbstractSchema.java | 4 +
.../drill/exec/store/StoragePluginRegistry.java | 48 ++++++++++-
.../drill/exec/store/SubSchemaWrapper.java | 87 ++++++++++++++++++++
.../exec/store/dfs/FileSystemSchemaFactory.java | 5 ++
.../store/hive/schema/HiveSchemaFactory.java | 4 +
.../drill/exec/store/ischema/OptiqProvider.java | 21 +++--
.../apache/drill/jdbc/test/TestJdbcQuery.java | 53 +++++++++---
7 files changed, 204 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d414d5dc/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
index 67b4ad9..485136b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
@@ -80,6 +80,10 @@ public abstract class AbstractSchema implements Schema{
throw new UnsupportedOperationException("New tables are not allowed in this schema");
}
+ public boolean showInInformationSchema() {
+ return true;
+ }
+
@Override
public Collection<Function> getFunctions(String name) {
return Collections.emptyList();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d414d5dc/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
index 1459cfa..b8d56ce 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
@@ -24,10 +24,12 @@ import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import com.google.common.collect.Lists;
import net.hydromatic.linq4j.expressions.DefaultExpression;
import net.hydromatic.linq4j.expressions.Expression;
import net.hydromatic.optiq.SchemaPlus;
@@ -43,7 +45,6 @@ import org.apache.drill.exec.cache.DistributedMap;
import org.apache.drill.exec.exception.DrillbitStartupException;
import org.apache.drill.exec.planner.logical.DrillRuleSets;
import org.apache.drill.exec.planner.logical.StoragePlugins;
-import org.apache.drill.exec.rpc.user.DrillUser;
import org.apache.drill.exec.rpc.user.UserSession;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.dfs.FileSystemPlugin;
@@ -229,6 +230,51 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage
for(Map.Entry<String, StoragePlugin> e : plugins.entrySet()){
e.getValue().registerSchemas(session, parent);
}
+
+ // Add second level schema as top level schema with name qualified with parent schema name
+ // Ex: "dfs" schema has "default" and "tmp" as sub schemas. Add following extra schemas "dfs.default" and
+ // "dfs.tmp" under root schema.
+ //
+ // Before change, schema tree looks like below:
+ // "root"
+ // -- "dfs"
+ // -- "default"
+ // -- "tmp"
+ // -- "hive"
+ // -- "default"
+ // -- "hivedb1"
+ //
+ // After the change, the schema tree looks like below:
+ // "root"
+ // -- "dfs"
+ // -- "default"
+ // -- "tmp"
+ // -- "dfs.default"
+ // -- "dfs.tmp"
+ // -- "hive"
+ // -- "default"
+ // -- "hivedb1"
+ // -- "hive.default"
+ // -- "hive.hivedb1"
+ List<SchemaPlus> secondLevelSchemas = Lists.newArrayList();
+ for(String firstLevelSchemaName : parent.getSubSchemaNames()) {
+ SchemaPlus firstLevelSchema = parent.getSubSchema(firstLevelSchemaName);
+ for(String secondLevelSchemaName : firstLevelSchema.getSubSchemaNames()) {
+ secondLevelSchemas.add(firstLevelSchema.getSubSchema(secondLevelSchemaName));
+ }
+ }
+
+ for(SchemaPlus schema : secondLevelSchemas) {
+ AbstractSchema drillSchema;
+ try {
+ drillSchema = schema.unwrap(AbstractSchema.class);
+ } catch(ClassCastException e) {
+ throw new RuntimeException(String.format("Schema '%s' is not expected under root schema", schema.getName()));
+ }
+
+ SubSchemaWrapper wrapper = new SubSchemaWrapper(drillSchema);
+ parent.add(wrapper.getName(), wrapper);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d414d5dc/exec/java-exec/src/main/java/org/apache/drill/exec/store/SubSchemaWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SubSchemaWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SubSchemaWrapper.java
new file mode 100644
index 0000000..576d761
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SubSchemaWrapper.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store;
+
+import com.google.common.collect.ImmutableList;
+import net.hydromatic.optiq.Function;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.Table;
+import org.apache.drill.exec.planner.logical.CreateTableEntry;
+
+import java.util.Collection;
+import java.util.Set;
+
+public class SubSchemaWrapper extends AbstractSchema {
+
+ private final AbstractSchema innerSchema;
+
+ public SubSchemaWrapper(AbstractSchema innerSchema) {
+ super(ImmutableList.<String>of(), innerSchema.getFullSchemaName());
+ this.innerSchema = innerSchema;
+ }
+
+ @Override
+ public boolean showInInformationSchema() {
+ return false;
+ }
+
+ @Override
+ public AbstractSchema getDefaultSchema() {
+ return innerSchema.getDefaultSchema();
+ }
+
+ @Override
+ public CreateTableEntry createNewTable(String tableName) {
+ return innerSchema.createNewTable(tableName);
+ }
+
+ @Override
+ public Collection<Function> getFunctions(String name) {
+ return innerSchema.getFunctions(name);
+ }
+
+ @Override
+ public Set<String> getFunctionNames() {
+ return innerSchema.getFunctionNames();
+ }
+
+ @Override
+ public Schema getSubSchema(String name) {
+ return innerSchema.getSubSchema(name);
+ }
+
+ @Override
+ public Set<String> getSubSchemaNames() {
+ return innerSchema.getSubSchemaNames();
+ }
+
+ @Override
+ public boolean isMutable() {
+ return innerSchema.isMutable();
+ }
+
+ @Override
+ public Table getTable(String name) {
+ return innerSchema.getTable(name);
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ return innerSchema.getTableNames();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d414d5dc/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
index 28496eb..9a57aa2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
@@ -86,6 +86,11 @@ public class FileSystemSchemaFactory implements SchemaFactory{
}
@Override
+ public boolean showInInformationSchema() {
+ return false;
+ }
+
+ @Override
public Table getTable(String name) {
return defaultSchema.getTable(name);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d414d5dc/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
index cff6bb1..6dd6143 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
@@ -212,6 +212,10 @@ public class HiveSchemaFactory implements SchemaFactory {
}
}
+ @Override
+ public boolean showInInformationSchema() {
+ return false;
+ }
@Override
public Set<String> getSubSchemaNames() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d414d5dc/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
index cb4f3d9..e12e5d6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
@@ -22,6 +22,7 @@ import net.hydromatic.optiq.SchemaPlus;
import net.hydromatic.optiq.Table;
import net.hydromatic.optiq.jdbc.JavaTypeFactoryImpl;
+import org.apache.drill.exec.store.AbstractSchema;
import org.eigenbase.reltype.RelDataType;
import org.eigenbase.reltype.RelDataTypeField;
import org.eigenbase.sql.type.SqlTypeName;
@@ -58,8 +59,8 @@ public class OptiqProvider {
*/
static public class Schemata extends Abstract {
@Override
- public boolean visitSchema(String schemaName, Schema schema) {
- if (schemaName != null && schemaName != "") {
+ public boolean visitSchema(String schemaName, SchemaPlus schema) {
+ if (shouldVisitSchema(schema) && schemaName != null && schemaName != "") {
writeRow("DRILL", schemaName, "<owner>");
}
return false;
@@ -173,12 +174,22 @@ public class OptiqProvider {
* If the schema visitor returns true, then visit the tables.
* If the table visitor returns true, then visit the fields (columns).
*/
- public boolean visitSchema(String schemaName, Schema schema){return true;}
+ public boolean visitSchema(String schemaName, SchemaPlus schema) {
+ return shouldVisitSchema(schema);
+ }
public boolean visitTableName(String schemaName, String tableName){return true;}
public boolean visitTable(String schemaName, String tableName, Table table){return true;}
public boolean visitField(String schemaName, String tableName, RelDataTypeField field){return true;}
-
+ protected boolean shouldVisitSchema(SchemaPlus schema) {
+ try {
+ AbstractSchema drillSchema = schema.unwrap(AbstractSchema.class);
+ return drillSchema.showInInformationSchema();
+ } catch(ClassCastException e) {
+ // ignore and return true as this is not a drill schema
+ }
+ return true;
+ }
/**
* Start scanning an Optiq Schema.
@@ -194,7 +205,7 @@ public class OptiqProvider {
* @param schema - the current schema.
* @param visitor - the methods to invoke at each entity in the schema.
*/
- private void scanSchema(String schemaPath, Schema schema) {
+ private void scanSchema(String schemaPath, SchemaPlus schema) {
// If we have an empty schema path, then don't insert a leading dot.
String separator;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d414d5dc/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index d087f7d..2d26b17 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -285,8 +285,6 @@ public class TestJdbcQuery extends JdbcTest{
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n" +
"TABLE_SCHEMA=hive.db1; TABLE_NAME=kv_db1\n" +
- "TABLE_SCHEMA=hive; TABLE_NAME=kv\n" +
- "TABLE_SCHEMA=hive; TABLE_NAME=foodate\n" +
"TABLE_SCHEMA=sys; TABLE_NAME=drillbits\n" +
"TABLE_SCHEMA=sys; TABLE_NAME=options\n" +
"TABLE_SCHEMA=INFORMATION_SCHEMA; TABLE_NAME=VIEWS\n" +
@@ -310,10 +308,10 @@ public class TestJdbcQuery extends JdbcTest{
);
JdbcAssert.withNoDefaultSchema()
- .sql("SHOW TABLES IN hive")
+ .sql("SHOW TABLES IN hive.`default`")
.returns(
- "TABLE_SCHEMA=hive; TABLE_NAME=kv\n" +
- "TABLE_SCHEMA=hive; TABLE_NAME=foodate\n");
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n");
}
@Test
@@ -335,13 +333,10 @@ public class TestJdbcQuery extends JdbcTest{
String expected =
"SCHEMA_NAME=hive.default\n" +
"SCHEMA_NAME=hive.db1\n" +
- "SCHEMA_NAME=hive\n" +
"SCHEMA_NAME=dfs.home\n" +
"SCHEMA_NAME=dfs.default\n" +
"SCHEMA_NAME=dfs.tmp\n" +
- "SCHEMA_NAME=dfs\n" +
"SCHEMA_NAME=cp.default\n" +
- "SCHEMA_NAME=cp\n" +
"SCHEMA_NAME=sys\n" +
"SCHEMA_NAME=INFORMATION_SCHEMA\n";
@@ -352,8 +347,8 @@ public class TestJdbcQuery extends JdbcTest{
@Test
public void testShowDatabasesWhere() throws Exception{
JdbcAssert.withNoDefaultSchema()
- .sql("SHOW DATABASES WHERE SCHEMA_NAME='dfs'")
- .returns("SCHEMA_NAME=dfs\n");
+ .sql("SHOW DATABASES WHERE SCHEMA_NAME='dfs.tmp'")
+ .returns("SCHEMA_NAME=dfs.tmp\n");
}
@Test
@@ -362,8 +357,7 @@ public class TestJdbcQuery extends JdbcTest{
.sql("SHOW DATABASES LIKE '%i%'")
.returns(
"SCHEMA_NAME=hive.default\n"+
- "SCHEMA_NAME=hive.db1\n"+
- "SCHEMA_NAME=hive\n"
+ "SCHEMA_NAME=hive.db1"
);
}
@@ -859,4 +853,39 @@ public class TestJdbcQuery extends JdbcTest{
}
});
}
+
+ // Tests using backticks around the complete schema path
+ // select * from `dfs.tmp`.`/tmp/nation.parquet`;
+ @Test
+ public void testCompleteSchemaRef1() throws Exception {
+ testQuery("select * from `cp.default`.`employee.json` limit 2");
+ }
+
+ @Test
+ public void testCompleteSchemaRef2() throws Exception {
+ JdbcAssert.withNoDefaultSchema().withConnection(new Function<Connection, Void>() {
+ public Void apply(Connection connection) {
+ try {
+ Statement statement = connection.createStatement();
+
+ // change default schema
+ ResultSet resultSet = statement.executeQuery("USE `dfs.default`");
+ String result = JdbcAssert.toString(resultSet).trim();
+ String expected = "ok=true; summary=Default schema changed to 'dfs.default'";
+ assertTrue(String.format("Generated string:\n%s\ndoes not match:\n%s", result, expected), expected.equals(result));
+
+ resultSet = statement.executeQuery(
+ String.format("select R_REGIONKEY from `%s/../../sample-data/region.parquet` LIMIT 1", WORKING_PATH));
+ result = JdbcAssert.toString(resultSet).trim();
+ expected = "R_REGIONKEY=0";
+ assertTrue(String.format("Generated string:\n%s\ndoes not match:\n%s", result, expected),
+ expected.equals(result));
+
+ return null;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
}
[2/7] git commit: DRILL-608: Fix casting to decimal data types with
difference scales.
Posted by ja...@apache.org.
DRILL-608: Fix casting to decimal data types with difference scales.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/b75782dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/b75782dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/b75782dc
Branch: refs/heads/master
Commit: b75782dc2851bd3ce224945604efe7c5f289c339
Parents: 2631bbc
Author: Mehant Baid <me...@gmail.com>
Authored: Mon May 12 17:53:17 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Mon May 12 21:33:20 2014 -0700
----------------------------------------------------------------------
.../drill/common/util/DecimalUtility.java | 102 +++++++++++++++++++
.../templates/Decimal/CastDecimalSimilar.java | 5 +
.../templates/Decimal/CastSrcDecimalSimple.java | 5 +
.../templates/Decimal/DecimalFunctions.java | 100 +-----------------
4 files changed, 113 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b75782dc/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
index 0cacc59..ff8725f 100644
--- a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
+++ b/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
@@ -493,5 +493,107 @@ public class DecimalUtility {
return new BigDecimal(unscaledValue, scale);
}
+ public static void roundDecimal(ByteBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
+ int newScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(desiredScale);
+ int origScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(currentScale);
+
+ if (desiredScale < currentScale) {
+
+ boolean roundUp = false;
+
+ //Extract the first digit to be truncated to check if we need to round up
+ int truncatedScaleIndex = desiredScale + 1;
+ if (truncatedScaleIndex <= currentScale) {
+ int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1;
+ extractDigitIndex += org.apache.drill.common.util.DecimalUtility.roundUp(truncatedScaleIndex);
+ int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex);
+ int temp = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS);
+ if (temp != 0) {
+ extractDigit = extractDigit / (int) (Math.pow(10, temp));
+ }
+ if ((extractDigit % 10) > 4) {
+ roundUp = true;
+ }
+ }
+
+ // Get the source index beyond which we will truncate
+ int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1;
+ int srcIndex = srcIntIndex + newScaleRoundedUp;
+
+ // Truncate the remaining fractional part, move the integer part
+ int destIndex = nDecimalDigits - 1;
+ if (srcIndex != destIndex) {
+ while (srcIndex >= 0) {
+ setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--));
+ }
+
+ // Set the remaining portion of the decimal to be zeroes
+ while (destIndex >= 0) {
+ setInteger(result, start, destIndex--, 0);
+ }
+ srcIndex = nDecimalDigits - 1;
+ }
+
+ // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
+ int truncateFactor = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS);
+ if (truncateFactor != org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) {
+ truncateFactor = (int) Math.pow(10, truncateFactor);
+ int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1);
+ fractionalDigits /= truncateFactor;
+ setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor);
+ }
+
+ // Finally round up the digit if needed
+ if (roundUp == true) {
+ srcIndex = nDecimalDigits - 1;
+ int carry;
+ if (truncateFactor != org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) {
+ carry = truncateFactor;
+ } else {
+ carry = 1;
+ }
+
+ while (srcIndex >= 0) {
+ int value = getIntegerFromSparseBuffer(result, start, srcIndex);
+ value += carry;
+
+ if (value >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
+ setInteger(result, start, srcIndex--, value % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
+ carry = value / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
+ } else {
+ setInteger(result, start, srcIndex--, value);
+ carry = 0;
+ break;
+ }
+ }
+ }
+ } else if (desiredScale > currentScale) {
+ // Add fractional digits to the decimal
+
+ // Check if we need to shift the decimal digits to the left
+ if (newScaleRoundedUp > origScaleRoundedUp) {
+ int srcIndex = 0;
+ int destIndex = newScaleRoundedUp - origScaleRoundedUp;
+
+ // Check while extending scale, we are not overwriting integer part
+ while (srcIndex < destIndex) {
+ if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) {
+ throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
+ }
+ }
+
+ srcIndex = 0;
+ while (destIndex < nDecimalDigits) {
+ setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++));
+ }
+
+ // Clear the remaining part
+ while (srcIndex < nDecimalDigits) {
+ setInteger(result, start, srcIndex++, 0);
+ }
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b75782dc/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
index 0d99569..a59cfdb 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
@@ -74,6 +74,11 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
for (; inputIdx >= 0; inputIdx--, outputIdx--) {
out.setInteger(outputIdx, in.getInteger(inputIdx));
}
+
+ // round up or down the scale
+ if (in.scale != out.scale) {
+ org.apache.drill.common.util.DecimalUtility.roundDecimal(out.buffer, out.start, out.nDecimalDigits, out.scale, in.scale);
+ }
}
}
</#if> <#-- type.major -->
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b75782dc/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
index 242fdb1..36a6ef3 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
@@ -208,6 +208,11 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
value = value/org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
index--;
}
+
+ // Round up or down the scale
+ if (in.scale != out.scale) {
+ org.apache.drill.common.util.DecimalUtility.roundDecimal(out.buffer, out.start, out.nDecimalDigits, out.scale, in.scale);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b75782dc/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
index 60b22fb..6cd6ade 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
@@ -913,105 +913,7 @@ public class ${type.name}Functions {
result.start = left.start;
result.sign = left.sign;
- int newScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(right.value);
- int origScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(left.scale);
-
- if (right.value < left.scale) {
-
- boolean roundUp = false;
-
- //Extract the first digit to be truncated to check if we need to round up
- int truncatedScaleIndex = right.value + 1;
- if (truncatedScaleIndex <= left.scale) {
- int extractDigitIndex = ${type.storage} - origScaleRoundedUp -1;
- extractDigitIndex += org.apache.drill.common.util.DecimalUtility.roundUp(truncatedScaleIndex);
- int extractDigit = result.getInteger(extractDigitIndex);
- int temp = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS);
- if (temp != 0) {
- extractDigit = extractDigit / (int) (Math.pow(10, temp));
- }
- if ((extractDigit % 10) > 4) {
- roundUp = true;
- }
- }
-
- // Get the source index beyond which we will truncate
- int srcIntIndex = ${type.storage} - origScaleRoundedUp - 1;
- int srcIndex = srcIntIndex + newScaleRoundedUp;
-
- // Truncate the remaining fractional part, move the integer part
- int destIndex = ${type.storage} - 1;
- if (srcIndex != destIndex) {
- while (srcIndex >= 0) {
- result.setInteger(destIndex--, result.getInteger(srcIndex--));
- }
-
- // Set the remaining portion of the decimal to be zeroes
- while (destIndex >= 0) {
- result.setInteger(destIndex--, 0);
- }
- srcIndex = ${type.storage} - 1;
- }
-
- // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
- int truncateFactor = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - (right.value % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS);
- if (truncateFactor != org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) {
- truncateFactor = (int) Math.pow(10, truncateFactor);
- int fractionalDigits = result.getInteger(${type.storage} - 1);
- fractionalDigits /= truncateFactor;
- result.setInteger(${type.storage} - 1, fractionalDigits * truncateFactor);
- }
-
- // Finally round up the digit if needed
- if (roundUp == true) {
- srcIndex = ${type.storage} - 1;
- int carry;
- if (truncateFactor != org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) {
- carry = truncateFactor;
- } else {
- carry = 1;
- }
-
- while (srcIndex >= 0) {
- int value = result.getInteger(srcIndex);
- value += carry;
-
- if (value >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- result.setInteger(srcIndex--, value % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
- carry = value / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
- } else {
- result.setInteger(srcIndex--, value);
- carry = 0;
- break;
- }
- }
- }
- } else if (right.value > left.scale) {
- // Add fractional digits to the decimal
-
- // Check if we need to shift the decimal digits to the left
- if (newScaleRoundedUp > origScaleRoundedUp) {
- int srcIndex = 0;
- int destIndex = newScaleRoundedUp - origScaleRoundedUp;
-
- // Check while extending scale, we are not overwriting integer part
- while (srcIndex < destIndex) {
- if (result.getInteger(srcIndex++) != 0) {
- throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
- }
- }
-
- srcIndex = 0;
- while (destIndex < ${type.storage}) {
- result.setInteger(srcIndex++, result.getInteger(destIndex++));
- }
-
- // Clear the remaining part
- while (srcIndex < ${type.storage}) {
- result.setInteger(srcIndex++, 0);
- }
- }
- }
+ org.apache.drill.common.util.DecimalUtility.roundDecimal(result.buffer, result.start, result.nDecimalDigits, result.scale, left.scale);
}
}
[6/7] git commit: DRILL-646: Broadcast-inner plans for Hash Join and
Merge Join.
Posted by ja...@apache.org.
DRILL-646: Broadcast-inner plans for Hash Join and Merge Join.
- Consolidated planning methods in base class.
- Add option to enable/disable broadcast join. Default is disabled for now.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/f1eaadea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/f1eaadea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/f1eaadea
Branch: refs/heads/master
Commit: f1eaadea87558bbb96bdc01fc69b78879fd65037
Parents: d414d5d
Author: Aman Sinha <as...@maprtech.com>
Authored: Sat May 10 18:05:14 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Mon May 12 21:33:36 2014 -0700
----------------------------------------------------------------------
.../exec/planner/physical/HashJoinPrule.java | 48 +++-----
.../exec/planner/physical/JoinPruleBase.java | 115 +++++++++++++++++++
.../exec/planner/physical/MergeJoinPrule.java | 37 +-----
.../exec/planner/physical/PlannerSettings.java | 12 +-
.../server/options/SystemOptionManager.java | 1 +
5 files changed, 146 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f1eaadea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrule.java
index 851877f..cbf1762 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrule.java
@@ -22,10 +22,13 @@ import java.util.logging.Logger;
import org.apache.drill.exec.planner.logical.DrillJoinRel;
import org.apache.drill.exec.planner.logical.RelOptHelper;
import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelCollation;
import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.metadata.RelMetadataQuery;
import org.eigenbase.relopt.RelOptRule;
import org.eigenbase.relopt.RelOptRuleCall;
import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.relopt.volcano.RelSubset;
import org.eigenbase.trace.EigenbaseTrace;
import com.google.common.collect.ImmutableList;
@@ -36,8 +39,7 @@ public class HashJoinPrule extends JoinPruleBase {
private HashJoinPrule() {
super(
- RelOptHelper.some(DrillJoinRel.class, RelOptHelper.any(RelNode.class), RelOptHelper.any(RelNode.class)),
- "Prel.HashJoinPrule");
+ RelOptHelper.any(DrillJoinRel.class), "Prel.HashJoinPrule");
}
@Override
@@ -48,50 +50,26 @@ public class HashJoinPrule extends JoinPruleBase {
@Override
public void onMatch(RelOptRuleCall call) {
final DrillJoinRel join = (DrillJoinRel) call.rel(0);
- final RelNode left = call.rel(1);
- final RelNode right = call.rel(2);
+ final RelNode left = join.getLeft();
+ final RelNode right = join.getRight();
if (!checkPreconditions(join, left, right)) {
return;
}
try {
- // Create transform request for HashJoin plan with both children HASH distributed
- DrillDistributionTrait hashLeftPartition = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(join.getLeftKeys())));
- DrillDistributionTrait hashRightPartition = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(join.getRightKeys())));
- RelTraitSet traitsLeft = left.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(hashLeftPartition);
- RelTraitSet traitsRight = right.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(hashRightPartition);
-
- createTransformRequest(call, join, left, right, traitsLeft, traitsRight);
-
- // Create transform request for HashJoin plan with left child ANY distributed and right child BROADCAST distributed
- /// TODO: ANY distribution seems to create some problems..need to revisit
- // DrillDistributionTrait distAnyLeft = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.ANY);
- DrillDistributionTrait distBroadcastRight = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.BROADCAST_DISTRIBUTED);
- // traitsLeft = left.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(distAnyLeft);
- traitsRight = right.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(distBroadcastRight);
-
- //temporarily not generate this plan
- //createTransformRequest(call, join, left, right, traitsLeft, traitsRight);
+ createDistBothPlan(call, join, PhysicalJoinType.HASH_JOIN, left, right, null /* left collation */, null /* right collation */);
+
+ if (checkBroadcastConditions(call.getPlanner(), join, left, right)) {
+ createBroadcastPlan(call, join, PhysicalJoinType.HASH_JOIN, left, right, null /* left collation */, null /* right collation */);
+ // createBroadcastPlan1(call, join, PhysicalJoinType.HASH_JOIN, left, right, null, null);
+ }
+
} catch (InvalidRelException e) {
tracer.warning(e.toString());
}
}
- private void createTransformRequest(RelOptRuleCall call, DrillJoinRel join,
- RelNode left, RelNode right,
- RelTraitSet traitsLeft, RelTraitSet traitsRight)
- throws InvalidRelException {
-
- final RelNode convertedLeft = convert(left, traitsLeft);
- final RelNode convertedRight = convert(right, traitsRight);
-
- HashJoinPrel newJoin = new HashJoinPrel(join.getCluster(), traitsLeft,
- convertedLeft, convertedRight, join.getCondition(),
- join.getJoinType());
- call.transformTo(newJoin) ;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f1eaadea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPruleBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPruleBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPruleBase.java
index 883aac5..1b30d68 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPruleBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPruleBase.java
@@ -20,19 +20,30 @@ package org.apache.drill.exec.planner.physical;
import java.util.List;
+import org.apache.drill.exec.planner.common.DrillJoinRelBase;
import org.apache.drill.exec.planner.logical.DrillJoinRel;
import org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelCollation;
import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.metadata.RelMetadataQuery;
+import org.eigenbase.relopt.RelOptPlanner;
import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
import org.eigenbase.relopt.RelOptRuleOperand;
import org.eigenbase.relopt.RelOptUtil;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.relopt.volcano.RelSubset;
import org.eigenbase.rex.RexNode;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
// abstract base class for the join physical rules
public abstract class JoinPruleBase extends RelOptRule {
+ protected static enum PhysicalJoinType {HASH_JOIN, MERGE_JOIN};
+
protected JoinPruleBase(RelOptRuleOperand operand, String description) {
super(operand, description);
}
@@ -63,4 +74,108 @@ public abstract class JoinPruleBase extends RelOptRule {
return distFields;
}
+ protected boolean checkBroadcastConditions(RelOptPlanner planner, DrillJoinRel join, RelNode left, RelNode right) {
+ if (! PrelUtil.getPlannerSettings(planner).isBroadcastJoinEnabled()) {
+ return false;
+ }
+
+ double estimatedRightRowCount = RelMetadataQuery.getRowCount(right);
+ if (estimatedRightRowCount < PrelUtil.getSettings(join.getCluster()).getBroadcastThreshold()
+ && ! left.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE).equals(DrillDistributionTrait.SINGLETON)
+ ) {
+ return true;
+ }
+ return false;
+ }
+
+ // Create join plan with both left and right children hash distributed. If the physical join type
+ // is MergeJoin, a collation must be provided for both left and right child and the plan will contain
+ // sort converter if necessary to provide the collation.
+ protected void createDistBothPlan(RelOptRuleCall call, DrillJoinRel join,
+ PhysicalJoinType physicalJoinType,
+ RelNode left, RelNode right,
+ RelCollation collationLeft, RelCollation collationRight) throws InvalidRelException {
+
+ DrillDistributionTrait hashLeftPartition = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(join.getLeftKeys())));
+ DrillDistributionTrait hashRightPartition = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(join.getRightKeys())));
+ RelTraitSet traitsLeft = null;
+ RelTraitSet traitsRight = null;
+
+ if (physicalJoinType == PhysicalJoinType.MERGE_JOIN) {
+ assert collationLeft != null && collationRight != null;
+ traitsLeft = left.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collationLeft).plus(hashLeftPartition);
+ traitsRight = right.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collationRight).plus(hashRightPartition);
+ } else if (physicalJoinType == PhysicalJoinType.HASH_JOIN) {
+ traitsLeft = left.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(hashLeftPartition);
+ traitsRight = right.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(hashRightPartition);
+ }
+
+ final RelNode convertedLeft = convert(left, traitsLeft);
+ final RelNode convertedRight = convert(right, traitsRight);
+
+ DrillJoinRelBase newJoin = null;
+
+ if (physicalJoinType == PhysicalJoinType.HASH_JOIN) {
+ newJoin = new HashJoinPrel(join.getCluster(), traitsLeft,
+ convertedLeft, convertedRight, join.getCondition(),
+ join.getJoinType());
+
+ } else if (physicalJoinType == PhysicalJoinType.MERGE_JOIN) {
+ newJoin = new MergeJoinPrel(join.getCluster(), traitsLeft,
+ convertedLeft, convertedRight, join.getCondition(),
+ join.getJoinType());
+ }
+ call.transformTo(newJoin);
+ }
+
+ // Create join plan with left child ANY distributed and right child BROADCAST distributed. If the physical join type
+ // is MergeJoin, a collation must be provided for both left and right child and the plan will contain sort converter
+ // if necessary to provide the collation.
+ protected void createBroadcastPlan(RelOptRuleCall call, DrillJoinRel join,
+ PhysicalJoinType physicalJoinType,
+ RelNode left, RelNode right,
+ RelCollation collationLeft, RelCollation collationRight) throws InvalidRelException {
+
+ DrillDistributionTrait distBroadcastRight = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.BROADCAST_DISTRIBUTED);
+ RelTraitSet traitsRight = null;
+ if (physicalJoinType == PhysicalJoinType.MERGE_JOIN) {
+ assert collationLeft != null && collationRight != null;
+ traitsRight = right.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collationRight).plus(distBroadcastRight);
+ } else {
+ traitsRight = right.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(distBroadcastRight);
+ }
+
+ RelTraitSet traitsLeft = left.getTraitSet().plus(Prel.DRILL_PHYSICAL);
+ RelNode convertedLeft = convert(left, traitsLeft);
+ RelNode convertedRight = convert(right, traitsRight);
+
+ traitsLeft = left.getTraitSet().plus(Prel.DRILL_PHYSICAL);
+
+ DrillJoinRelBase newJoin = null;
+
+ if (convertedLeft instanceof RelSubset) {
+ RelSubset subset = (RelSubset) convertedLeft;
+ for (RelNode rel : subset.getRelList()) {
+ if (!rel.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE).equals(DrillDistributionTrait.DEFAULT)) {
+ DrillDistributionTrait toDist = rel.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE);
+ if (physicalJoinType == PhysicalJoinType.MERGE_JOIN) {
+ traitsLeft = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collationLeft).plus(toDist);
+ } else {
+ traitsLeft = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(toDist);
+ }
+
+ RelNode newLeft = convert(left, traitsLeft);
+ if (physicalJoinType == PhysicalJoinType.HASH_JOIN) {
+ newJoin = new HashJoinPrel(join.getCluster(), traitsLeft, newLeft, convertedRight, join.getCondition(),
+ join.getJoinType());
+ } else if (physicalJoinType == PhysicalJoinType.MERGE_JOIN) {
+ newJoin = new MergeJoinPrel(join.getCluster(), traitsLeft, newLeft, convertedRight, join.getCondition(),
+ join.getJoinType());
+ }
+ call.transformTo(newJoin) ;
+ }
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f1eaadea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrule.java
index 30f651c..a5be5f8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrule.java
@@ -27,9 +27,11 @@ import org.eigenbase.rel.RelCollation;
import org.eigenbase.rel.RelCollationImpl;
import org.eigenbase.rel.RelFieldCollation;
import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.metadata.RelMetadataQuery;
import org.eigenbase.relopt.RelOptRule;
import org.eigenbase.relopt.RelOptRuleCall;
import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.relopt.volcano.RelSubset;
import org.eigenbase.trace.EigenbaseTrace;
import com.google.common.collect.ImmutableList;
@@ -64,43 +66,16 @@ public class MergeJoinPrule extends JoinPruleBase {
RelCollation collationLeft = getCollation(join.getLeftKeys());
RelCollation collationRight = getCollation(join.getRightKeys());
- // Create transform request for MergeJoin plan with both children HASH distributed
- DrillDistributionTrait hashLeftPartition = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(join.getLeftKeys())));
- DrillDistributionTrait hashRightPartition = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(join.getRightKeys())));
+ createDistBothPlan(call, join, PhysicalJoinType.MERGE_JOIN, left, right, collationLeft, collationRight);
- RelTraitSet traitsLeft = left.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collationLeft).plus(hashLeftPartition);
- RelTraitSet traitsRight = right.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collationRight).plus(hashRightPartition);
-
- createTransformRequest(call, join, left, right, traitsLeft, traitsRight);
-
- // Create transform request for MergeJoin plan with left child ANY distributed and right child BROADCAST distributed
- /// TODO: ANY distribution seems to create some problems..need to revisit
- // DrillDistributionTrait distAnyLeft = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.ANY);
- DrillDistributionTrait distBroadcastRight = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.BROADCAST_DISTRIBUTED);
- // traitsLeft = left.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collationLeft).plus(distAnyLeft);
- traitsRight = right.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collationRight).plus(distBroadcastRight);
-
- //temporarily not generate this plan
- //createTransformRequest(call, join, left, right, traitsLeft, traitsRight);
+ if (checkBroadcastConditions(call.getPlanner(), join, left, right)) {
+ createBroadcastPlan(call, join, PhysicalJoinType.MERGE_JOIN, left, right, collationLeft, collationRight);
+ }
} catch (InvalidRelException e) {
tracer.warning(e.toString());
}
}
-
- private void createTransformRequest(RelOptRuleCall call, DrillJoinRel join,
- RelNode left, RelNode right,
- RelTraitSet traitsLeft, RelTraitSet traitsRight)
- throws InvalidRelException {
-
- final RelNode convertedLeft = convert(left, traitsLeft);
- final RelNode convertedRight = convert(right, traitsRight);
-
- MergeJoinPrel newJoin = new MergeJoinPrel(join.getCluster(), traitsLeft,
- convertedLeft, convertedRight, join.getCondition(),
- join.getJoinType());
- call.transformTo(newJoin) ;
- }
private RelCollation getCollation(List<Integer> keys){
List<RelFieldCollation> fields = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f1eaadea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index 5bead30..e65ef17 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -28,6 +28,7 @@ public class PlannerSettings implements FrameworkContext{
private int numEndPoints = 0;
private boolean useDefaultCosting = false; // True: use default Optiq costing, False: use Drill costing
+ private int broadcastThreshold = 10000; // Consider broadcast inner plans if estimated rows is less than this threshold
public static final OptionValidator EXCHANGE = new BooleanValidator("planner.disable_exchanges", false);
public static final OptionValidator HASHAGG = new BooleanValidator("planner.enable_hashagg", true);
@@ -35,7 +36,8 @@ public class PlannerSettings implements FrameworkContext{
public static final OptionValidator HASHJOIN = new BooleanValidator("planner.enable_hashjoin", true);
public static final OptionValidator MERGEJOIN = new BooleanValidator("planner.enable_mergejoin", true);
public static final OptionValidator MULTIPHASE = new BooleanValidator("planner.enable_multiphase_agg", false);
-
+ public static final OptionValidator BROADCAST = new BooleanValidator("planner.enable_broadcast_join", false);
+
public OptionManager options = null;
public PlannerSettings(OptionManager options){
@@ -82,6 +84,14 @@ public class PlannerSettings implements FrameworkContext{
return options.getOption(MULTIPHASE.getOptionName()).bool_val;
}
+ public boolean isBroadcastJoinEnabled() {
+ return options.getOption(BROADCAST.getOptionName()).bool_val;
+ }
+
+ public int getBroadcastThreshold() {
+ return broadcastThreshold;
+ }
+
@Override
public <T> T unwrap(Class<T> clazz) {
if(clazz == PlannerSettings.class){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f1eaadea/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 21031e5..8d9a68f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -40,6 +40,7 @@ public class SystemOptionManager implements OptionManager{
PlannerSettings.HASHJOIN,
PlannerSettings.MERGEJOIN,
PlannerSettings.MULTIPHASE,
+ PlannerSettings.BROADCAST,
ExecConstants.OUTPUT_FORMAT_VALIDATOR,
ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR
};
[7/7] git commit: Disabling intermittently failing TestParquetWriter
Posted by ja...@apache.org.
Disabling intermittently failing TestParquetWriter
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e7a486d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e7a486d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e7a486d7
Branch: refs/heads/master
Commit: e7a486d784c072458d44b7692ea0262da368f001
Parents: f1eaade
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Tue May 13 02:11:51 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Tue May 13 02:11:51 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/physical/impl/writer/TestParquetWriter.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e7a486d7/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index d9fa722..757bab7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -35,6 +35,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+@Ignore
public class TestParquetWriter extends BaseTestQuery {
static FileSystem fs;