You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by su...@apache.org on 2016/12/13 00:39:58 UTC
[01/14] drill git commit: DRILL-5091: Handle new Java 8 JDBC
functions and add missing test config parameter
Repository: drill
Updated Branches:
refs/heads/master 351dea61b -> cf2b7c70e
DRILL-5091: Handle new Java 8 JDBC functions and add missing test config parameter
* JDBC unit test fails to set up test storage plugin config on Java 8
* Cleans up compiler warnings seen in Eclipse.
* Added TODO's based on code reviews
closes #676
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a6a0f2f0
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a6a0f2f0
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a6a0f2f0
Branch: refs/heads/master
Commit: a6a0f2f070ff48d622916beb97d6da6f0b5faad1
Parents: d9077a0
Author: Paul Rogers <pr...@maprtech.com>
Authored: Fri Dec 2 14:02:03 2016 -0800
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:01 2016 -0800
----------------------------------------------------------------------
.../org/apache/drill/exec/server/Drillbit.java | 2 +-
.../exec/store/StoragePluginRegistryImpl.java | 8 +++
.../exec/store/sys/PersistentStoreConfig.java | 2 +-
.../store/sys/store/LocalPersistentStore.java | 17 ++---
.../CachingPersistentStoreProvider.java | 3 +-
.../provider/LocalPersistentStoreProvider.java | 8 +--
.../org/apache/drill/jdbc/JdbcTestBase.java | 7 +-
...l2489CallsAfterCloseThrowExceptionsTest.java | 71 +++++++++++++-------
...69UnsupportedReportsUseSqlExceptionTest.java | 66 ++++++++++--------
.../org/apache/drill/jdbc/test/JdbcAssert.java | 8 ++-
10 files changed, 119 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a0f2f0/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
index 3f74268..a28dc91 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
@@ -101,7 +101,7 @@ public class Drillbit implements AutoCloseable {
storeProvider = new CachingPersistentStoreProvider(new LocalPersistentStoreProvider(config));
} else {
coord = new ZKClusterCoordinator(config);
- storeProvider = new PersistentStoreRegistry(this.coord, config).newPStoreProvider();
+ storeProvider = new PersistentStoreRegistry<ClusterCoordinator>(this.coord, config).newPStoreProvider();
isDistributedMode = true;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a0f2f0/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
index ad38586..bf4affd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
@@ -115,10 +115,12 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
});
}
+ @Override
public PersistentStore<StoragePluginConfig> getStore() {
return pluginSystemTable;
}
+ @Override
public void init() throws DrillbitStartupException {
availablePlugins = findAvailablePlugins(classpathScan);
@@ -188,6 +190,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
plugins.put(name, plugin);
}
+ @Override
public void deletePlugin(String name) {
StoragePlugin plugin = plugins.remove(name);
closePlugin(plugin);
@@ -206,6 +209,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
}
}
+ @Override
public StoragePlugin createOrUpdate(String name, StoragePluginConfig config, boolean persist)
throws ExecutionSetupException {
for (;;) {
@@ -243,6 +247,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
}
}
+ @Override
public StoragePlugin getPlugin(String name) throws ExecutionSetupException {
StoragePlugin plugin = plugins.get(name);
if (name.equals(SYS_PLUGIN) || name.equals(INFORMATION_SCHEMA_PLUGIN)) {
@@ -267,6 +272,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
}
+ @Override
public StoragePlugin getPlugin(StoragePluginConfig config) throws ExecutionSetupException {
if (config instanceof NamedStoragePluginConfig) {
return getPlugin(((NamedStoragePluginConfig) config).name);
@@ -293,6 +299,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
}
}
+ @Override
public FormatPlugin getFormatPlugin(StoragePluginConfig storageConfig, FormatPluginConfig formatConfig)
throws ExecutionSetupException {
StoragePlugin p = getPlugin(storageConfig);
@@ -332,6 +339,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
return plugins.iterator();
}
+ @Override
public SchemaFactory getSchemaFactory() {
return schemaFactory;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a0f2f0/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreConfig.java
index ca319f2..00a75a2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreConfig.java
@@ -65,7 +65,7 @@ public class PersistentStoreConfig<V> {
@Override
public boolean equals(Object obj) {
if (obj instanceof PersistentStoreConfig) {
- final PersistentStoreConfig other = PersistentStoreConfig.class.cast(obj);
+ final PersistentStoreConfig<?> other = PersistentStoreConfig.class.cast(obj);
return Objects.equal(name, other.name)
&& Objects.equal(valueSerializer, other.valueSerializer)
&& Objects.equal(mode, other.mode);
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a0f2f0/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java
index 1ef8d12..b9a4b59 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java
@@ -31,27 +31,25 @@ import java.util.Map.Entry;
import javax.annotation.Nullable;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.drill.common.collections.ImmutableEntry;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
import org.apache.drill.exec.store.sys.BasePersistentStore;
-import org.apache.drill.exec.store.sys.PersistentStore;
import org.apache.drill.exec.store.sys.PersistentStoreConfig;
import org.apache.drill.exec.store.sys.PersistentStoreMode;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
public class LocalPersistentStore<V> extends BasePersistentStore<V> {
- private static final Logger logger = LoggerFactory.getLogger(LocalPersistentStore.class);
+// private static final Logger logger = LoggerFactory.getLogger(LocalPersistentStore.class);
private final Path basePath;
private final PersistentStoreConfig<V> config;
@@ -140,6 +138,7 @@ public class LocalPersistentStore<V> extends BasePersistentStore<V> {
return path;
}
+ @Override
public V get(String key) {
try{
Path path = makePath(key);
@@ -158,6 +157,7 @@ public class LocalPersistentStore<V> extends BasePersistentStore<V> {
}
}
+ @Override
public void put(String key, V value) {
try (OutputStream os = fs.create(makePath(key))) {
IOUtils.write(config.getSerializer().serialize(value), os);
@@ -181,6 +181,7 @@ public class LocalPersistentStore<V> extends BasePersistentStore<V> {
}
}
+ @Override
public void delete(String key) {
try {
fs.delete(makePath(key), false);
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a0f2f0/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/CachingPersistentStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/CachingPersistentStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/CachingPersistentStoreProvider.java
index 99ccc8e..771005f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/CachingPersistentStoreProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/CachingPersistentStoreProvider.java
@@ -29,7 +29,7 @@ import org.apache.drill.exec.store.sys.PersistentStoreConfig;
import org.apache.drill.exec.store.sys.PersistentStoreProvider;
public class CachingPersistentStoreProvider extends BasePersistentStoreProvider {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CachingPersistentStoreProvider.class);
+// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CachingPersistentStoreProvider.class);
private final ConcurrentMap<PersistentStoreConfig<?>, PersistentStore<?>> storeCache = Maps.newConcurrentMap();
private final PersistentStoreProvider provider;
@@ -38,6 +38,7 @@ public class CachingPersistentStoreProvider extends BasePersistentStoreProvider
this.provider = provider;
}
+ @Override
@SuppressWarnings("unchecked")
public <V> PersistentStore<V> getOrCreateStore(final PersistentStoreConfig<V> config) throws StoreException {
final PersistentStore<?> store = storeCache.get(config);
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a0f2f0/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/LocalPersistentStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/LocalPersistentStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/LocalPersistentStoreProvider.java
index 9bf18ab..0b4a201 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/LocalPersistentStoreProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/LocalPersistentStoreProvider.java
@@ -23,20 +23,18 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.StoreException;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
-import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
import org.apache.drill.exec.store.sys.PersistentStore;
import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
import org.apache.drill.exec.store.sys.store.LocalPersistentStore;
import org.apache.drill.exec.testing.store.NoWriteLocalStore;
import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* A really simple provider that stores data in the local file system, one value per file.
*/
public class LocalPersistentStoreProvider extends BasePersistentStoreProvider {
- private static final Logger logger = LoggerFactory.getLogger(LocalPersistentStoreProvider.class);
+// private static final Logger logger = LoggerFactory.getLogger(LocalPersistentStoreProvider.class);
private final Path path;
private final DrillFileSystem fs;
@@ -44,7 +42,7 @@ public class LocalPersistentStoreProvider extends BasePersistentStoreProvider {
// how to handle this flag.
private final boolean enableWrite;
- public LocalPersistentStoreProvider(final PersistentStoreRegistry registry) throws StoreException {
+ public LocalPersistentStoreProvider(final PersistentStoreRegistry<?> registry) throws StoreException {
this(registry.getConfig());
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a0f2f0/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
index 7fb601d..b9e8392 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
@@ -114,7 +114,8 @@ public class JdbcTestBase extends ExecTest {
protected static void changeSchema(Connection conn, String schema) {
final String query = String.format("use %s", schema);
- try ( Statement s = conn.createStatement() ) {
+ try (Statement s = conn.createStatement()) {
+ @SuppressWarnings("unused")
ResultSet r = s.executeQuery(query);
// TODO: Purge nextUntilEnd(...) and calls when remaining fragment
// race conditions are fixed (not just DRILL-2245 fixes).
@@ -147,10 +148,10 @@ public class JdbcTestBase extends ExecTest {
* (Note: Not a guaranteed test--depends on order in which test methods are
* run.)
*/
- @Ignore( "Usually disabled; enable temporarily to check tests" )
+ @Ignore("Usually disabled; enable temporarily to check tests")
@Test
public void testJdbcTestConnectionResettingCompatibility() {
- fail( "Intentional failure--did other test methods still run?" );
+ fail("Intentional failure--did other test methods still run?");
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a0f2f0/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2489CallsAfterCloseThrowExceptionsTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2489CallsAfterCloseThrowExceptionsTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2489CallsAfterCloseThrowExceptionsTest.java
index ee94fd2..b8a7895 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2489CallsAfterCloseThrowExceptionsTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2489CallsAfterCloseThrowExceptionsTest.java
@@ -38,6 +38,7 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLClientInfoException;
import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
import java.sql.Statement;
import java.sql.Struct;
import java.util.ArrayList;
@@ -81,7 +82,6 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
private static ResultSetMetaData resultSetMetaDataOfClosedStmt;
private static DatabaseMetaData databaseMetaDataOfClosedConn;
-
@BeforeClass
public static void setUpClosedObjects() throws Exception {
// (Note: Can't use JdbcTest's connect(...) for this test class.)
@@ -135,15 +135,14 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
resultSetMetaDataOfClosedStmt = rsmdForClosedStmt;
databaseMetaDataOfClosedConn = dbmd;
-
// Self-check that member variables are set (and objects are in right open
// or closed state):
- assertTrue( "Test setup error", closedConn.isClosed());
+ assertTrue("Test setup error", closedConn.isClosed());
assertFalse("Test setup error", openConn.isClosed());
- assertTrue( "Test setup error", closedPlainStmtOfOpenConn.isClosed());
- assertTrue( "Test setup error", closedPreparedStmtOfOpenConn.isClosed());
- assertTrue( "Test setup error", closedResultSetOfClosedStmt.isClosed());
- assertTrue( "Test setup error", closedResultSetOfOpenStmt.isClosed());
+ assertTrue("Test setup error", closedPlainStmtOfOpenConn.isClosed());
+ assertTrue("Test setup error", closedPreparedStmtOfOpenConn.isClosed());
+ assertTrue("Test setup error", closedResultSetOfClosedStmt.isClosed());
+ assertTrue("Test setup error", closedResultSetOfOpenStmt.isClosed());
// (No ResultSetMetaData.isClosed() or DatabaseMetaData.isClosed():)
assertNotNull("Test setup error", resultSetMetaDataOfClosedResultSet);
assertNotNull("Test setup error", resultSetMetaDataOfClosedStmt);
@@ -155,7 +154,6 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
openConn.close();
}
-
///////////////////////////////////////////////////////////////
// 1. Check that isClosed() and close() do not throw, and isClosed() returns
// true.
@@ -200,12 +198,10 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
assertThat(closedResultSetOfOpenStmt.isClosed(), equalTo(true));
}
-
///////////////////////////////////////////////////////////////
// 2. Check that all methods throw or not appropriately (either as specified
// by JDBC or currently intended as partial Avatica workaround).
-
/**
* Reflection-based checker of throwing of "already closed" exception by JDBC
* interfaces' implementation methods.
@@ -405,17 +401,14 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
+ ")";
return report;
}
-
} // class ThrowsClosedChecker<INTF>
-
private static class ClosedConnectionChecker
extends ThrowsClosedBulkChecker<Connection> {
private static final String STATEMENT_CLOSED_MESSAGE =
"Connection is already closed.";
-
ClosedConnectionChecker(Class<Connection> intf, Connection jdbcObject) {
super(intf, jdbcObject, STATEMENT_CLOSED_MESSAGE);
}
@@ -449,7 +442,6 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
}
return result;
}
-
} // class ClosedConnectionChecker
@Test
@@ -465,7 +457,6 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
}
}
-
private static class ClosedPlainStatementChecker
extends ThrowsClosedBulkChecker<Statement> {
@@ -477,11 +468,24 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
}
@Override
+ protected boolean isOkayNonthrowingMethod(Method method) {
+ // TODO: Java 8 method
+ if ("getLargeUpdateCount".equals(method.getName())) {
+ return true; }
+ return super.isOkayNonthrowingMethod(method);
+ }
+
+ @Override
protected boolean isOkaySpecialCaseException(Method method, Throwable cause) {
final boolean result;
if (super.isOkaySpecialCaseException(method, cause)) {
result = true;
}
+ else if ( method.getName().equals("executeLargeBatch")
+ || method.getName().equals("executeLargeUpdate")) {
+ // TODO: New Java 8 methods not implemented in Avatica.
+ result = true;
+ }
else if (RuntimeException.class == cause.getClass()
&& normalClosedExceptionText.equals(cause.getMessage())
&& (false
@@ -489,6 +493,7 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
|| method.getName().equals("getFetchDirection")
|| method.getName().equals("getFetchSize")
|| method.getName().equals("getMaxRows")
+ || method.getName().equals("getLargeMaxRows") // TODO: Java 8
)) {
// Special good-enough case--we had to use RuntimeException for now.
result = true;
@@ -498,7 +503,6 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
}
return result;
}
-
} // class ClosedPlainStatementChecker
@Test
@@ -513,7 +517,6 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
}
}
-
private static class ClosedPreparedStatementChecker
extends ThrowsClosedBulkChecker<PreparedStatement> {
@@ -526,6 +529,15 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
}
@Override
+ protected boolean isOkayNonthrowingMethod(Method method) {
+ // TODO: Java 8 methods not yet supported by Avatica.
+ if (method.getName().equals("getLargeUpdateCount")) {
+ return true;
+ }
+ return super.isOkayNonthrowingMethod(method);
+ }
+
+ @Override
protected boolean isOkaySpecialCaseException(Method method, Throwable cause) {
final boolean result;
if (super.isOkaySpecialCaseException(method, cause)) {
@@ -543,12 +555,19 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
// Special good-enough case--we had to use RuntimeException for now.
result = true;
}
+ else if ( method.getName().equals("setObject")
+ || method.getName().equals("executeLargeUpdate")
+ || method.getName().equals("executeLargeBatch")
+ || method.getName().equals("getLargeMaxRows")
+ ) {
+ // TODO: Java 8 methods not yet supported by Avatica.
+ result = true;
+ }
else {
result = false;
}
return result;
}
-
} // class closedPreparedStmtOfOpenConnChecker
@Test
@@ -564,7 +583,6 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
}
}
-
private static class ClosedResultSetChecker
extends ThrowsClosedBulkChecker<ResultSet> {
@@ -587,12 +605,16 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
// Special good-enough case--we had to use RuntimeException for now.
result = true;
}
+ else if (SQLFeatureNotSupportedException.class == cause.getClass()
+ && (method.getName().equals("updateObject"))) {
+ // TODO: Java 8 methods not yet supported by Avatica.
+ result = true;
+ }
else {
result = false;
}
return result;
}
-
} // class ClosedResultSetChecker
@Test
@@ -630,7 +652,6 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
ResultSetMetaData jdbcObject) {
super(intf, jdbcObject, RESULTSETMETADATA_CLOSED_MESSAGE);
}
-
} // class ClosedResultSetMetaDataChecker
@Test
@@ -671,12 +692,16 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
super(intf, jdbcObject, DATABASEMETADATA_CLOSED_MESSAGE);
}
+ @Override
protected boolean isOkayNonthrowingMethod(Method method) {
return
super.isOkayNonthrowingMethod(method)
|| method.getName().equals("getDriverMajorVersion")
|| method.getName().equals("getDriverMinorVersion")
- || method.getName().equals("getConnection");
+ || method.getName().equals("getConnection")
+ // TODO: New Java 8 methods not implemented in Avatica.
+ || method.getName().equals("getMaxLogicalLobSize")
+ || method.getName().equals("supportsRefCursors");
}
@Override
@@ -696,7 +721,6 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
}
return result;
}
-
} // class ClosedDatabaseMetaDataChecker
@@ -712,5 +736,4 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
fail("Already-closed exception error(s): \n" + checker.getReport());
}
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a0f2f0/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2769UnsupportedReportsUseSqlExceptionTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2769UnsupportedReportsUseSqlExceptionTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2769UnsupportedReportsUseSqlExceptionTest.java
index 8bc03aa..1de737a 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2769UnsupportedReportsUseSqlExceptionTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2769UnsupportedReportsUseSqlExceptionTest.java
@@ -17,38 +17,33 @@
*/
package org.apache.drill.jdbc.test;
-import static org.junit.Assert.*;
-import static org.hamcrest.CoreMatchers.*;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.slf4j.Logger;
-
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
import static org.slf4j.LoggerFactory.getLogger;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
-import java.sql.Array;
-import java.sql.CallableStatement;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
-import java.sql.SQLClientInfoException;
import java.sql.SQLException;
import java.sql.Statement;
-import java.sql.Struct;
import java.util.ArrayList;
import java.util.List;
import org.apache.drill.common.util.TestTools;
+import org.apache.drill.jdbc.AlreadyClosedSqlException;
import org.apache.drill.jdbc.Driver;
import org.apache.drill.jdbc.JdbcTestBase;
-import org.apache.drill.jdbc.AlreadyClosedSqlException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.slf4j.Logger;
/**
@@ -84,7 +79,6 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
private static ResultSetMetaData resultSetMetaData;
private static DatabaseMetaData databaseMetaData;
-
@BeforeClass
public static void setUpObjects() throws Exception {
// (Note: Can't use JdbcTest's connect(...) for this test class.)
@@ -117,7 +111,6 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
resultSetMetaData = resultSet.getMetaData();
databaseMetaData = connection.getMetaData();
-
// Self-check that member variables are set:
assertFalse("Test setup error", connection.isClosed());
assertFalse("Test setup error", plainStatement.isClosed());
@@ -133,7 +126,6 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
connection.close();
}
-
/**
* Reflection-based checker that exceptions thrown by JDBC interfaces'
* implementation methods for unsupported-operation cases are SQLExceptions
@@ -148,7 +140,6 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
private final StringBuilder failureLinesBuf = new StringBuilder();
private final StringBuilder successLinesBuf = new StringBuilder();
-
NoNonSqlExceptionsChecker(final Class<INTF> jdbcIntf,
final INTF jdbcObject) {
this.jdbcIntf = jdbcIntf;
@@ -280,6 +271,10 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
// code implements them.
successLinesBuf.append(resultLine);
}
+ else if (isOkaySpecialCaseException(method, cause)) {
+ successLinesBuf.append(resultLine);
+ }
+
else {
final String badResultLine =
"- " + methodLabel + " threw <" + cause + "> instead"
@@ -319,6 +314,15 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
}
}
+ /**
+ * Reports whether it's okay if given method throw given exception (that is
+ * not preferred AlreadyClosedException with regular message).
+ */
+ protected boolean isOkaySpecialCaseException(Method method,
+ Throwable cause) {
+ return false;
+ }
+
public boolean hadAnyFailures() {
return 0 != failureLinesBuf.length();
}
@@ -340,10 +344,8 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
+ ")";
return report;
}
-
} // class NoNonSqlExceptionsChecker<INTF>
-
@Test
public void testConnectionMethodsThrowRight() {
NoNonSqlExceptionsChecker<Connection> checker =
@@ -357,7 +359,6 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
}
}
-
private static class PlainStatementChecker
extends NoNonSqlExceptionsChecker<Statement> {
@@ -368,10 +369,18 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
this.factoryConnection = factoryConnection;
}
+ @Override
protected Statement getJdbcObject() throws SQLException {
return factoryConnection.createStatement();
}
+ @Override
+ protected boolean isOkaySpecialCaseException(Method method,
+ Throwable cause) {
+ // New Java 8 method not supported by Avatica
+
+ return method.getName().equals( "executeLargeBatch" );
+ }
} // class PlainStatementChecker
@Test
@@ -386,7 +395,6 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
}
}
-
private static class PreparedStatementChecker
extends NoNonSqlExceptionsChecker<PreparedStatement> {
@@ -397,10 +405,18 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
this.factoryConnection = factoryConnection;
}
+ @Override
protected PreparedStatement getJdbcObject() throws SQLException {
return factoryConnection.prepareStatement("VALUES 1");
}
+ @Override
+ protected boolean isOkaySpecialCaseException(Method method,
+ Throwable cause) {
+ // New Java 8 method not supported by Avatica
+
+ return method.getName().equals( "executeLargeBatch" );
+ }
} // class PlainStatementChecker
@Test
@@ -415,7 +431,6 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
}
}
-
@Test
public void testResultSetMethodsThrowRight() {
NoNonSqlExceptionsChecker<ResultSet> checker =
@@ -428,7 +443,6 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
}
}
-
@Test
public void testResultSetMetaDataMethodsThrowRight() {
NoNonSqlExceptionsChecker<ResultSetMetaData> checker =
@@ -442,7 +456,6 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
}
}
-
@Test
public void testDatabaseMetaDataMethodsThrowRight() {
NoNonSqlExceptionsChecker<DatabaseMetaData> checker =
@@ -455,5 +468,4 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
fail("Non-SQLException exception error(s): \n" + checker.getReport());
}
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a0f2f0/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
index 3f8514e..caa09b8 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
@@ -66,6 +66,11 @@ public class JdbcAssert {
public static Properties getDefaultProperties() {
final Properties properties = new Properties();
properties.setProperty("drillJDBCUnitTests", "true");
+
+ // Must set this to false to ensure that the tests ignore any existing
+ // plugin configurations stored in /tmp/drill.
+
+ properties.setProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, "false");
properties.setProperty(ExecConstants.HTTP_ENABLE, "false");
return properties;
}
@@ -246,7 +251,6 @@ public class JdbcAssert {
}
}
-
/**
* Checks that the current SQL statement returns the expected result lines. Lines are compared unordered; the test
* succeeds if the query returns these lines in any order.
@@ -291,7 +295,6 @@ public class JdbcAssert {
connection.close();
}
}
-
}
private SortedSet<String> unsortedList(List<String> strings) {
@@ -353,7 +356,6 @@ public class JdbcAssert {
private static interface ConnectionFactoryAdapter {
Connection createConnection() throws Exception;
}
-
}
// End JdbcAssert.java
[07/14] drill git commit: DRILL-5085: Update description for dynamic
UDFs directories in drill-env.sh and drill-module.conf
Posted by su...@apache.org.
DRILL-5085: Update description for dynamic UDFs directories in drill-env.sh and drill-module.conf
closes #672
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3cd2f445
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3cd2f445
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3cd2f445
Branch: refs/heads/master
Commit: 3cd2f4453216436d3f742773c4a4c78a5984c363
Parents: a6a0f2f
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Wed Nov 30 15:30:35 2016 +0000
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:02 2016 -0800
----------------------------------------------------------------------
distribution/src/resources/drill-env.sh | 7 ++++-
.../src/resources/drill-override-example.conf | 16 ++++++++++-
.../org/apache/drill/exec/ExecConstants.java | 2 +-
.../expr/fn/FunctionImplementationRegistry.java | 15 ++++++-----
.../fn/registry/RemoteFunctionRegistry.java | 13 ++++-----
.../src/main/resources/drill-module.conf | 28 ++++++++++----------
.../org/apache/drill/TestDynamicUDFSupport.java | 1 +
7 files changed, 52 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/distribution/src/resources/drill-env.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-env.sh b/distribution/src/resources/drill-env.sh
index cd11b47..62dffe6 100644
--- a/distribution/src/resources/drill-env.sh
+++ b/distribution/src/resources/drill-env.sh
@@ -141,4 +141,9 @@
# Arguments passed to sqlline (the Drill shell) at all times: whether
# Drill is embedded in Sqlline or not.
-#export DRILL_SHELL_JAVA_OPTS="..."
\ No newline at end of file
+#export DRILL_SHELL_JAVA_OPTS="..."
+
+# Location Drill should use for temporary files, such as downloaded dynamic UDFs jars.
+# Set to "/tmp" by default.
+#
+# export DRILL_TMP_DIR="..."
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/distribution/src/resources/drill-override-example.conf
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-override-example.conf b/distribution/src/resources/drill-override-example.conf
index 4be4aa2..f9d27b3 100644
--- a/distribution/src/resources/drill-override-example.conf
+++ b/distribution/src/resources/drill-override-example.conf
@@ -170,7 +170,21 @@ drill.exec: {
threadpool_size: 8,
decode_threadpool_size: 1
},
- debug.error_on_leak: true
+ debug.error_on_leak: true,
+ # Settings for Dynamic UDFs.
+ # See https://gist.github.com/arina-ielchiieva/a1c4cfa3890145c5ecb1b70a39cbff55#file-dynamicudfssupport-md.
+ udf: {
+ # number of retry attempts to update remote function registry
+ # if registry version was changed during update
+ retry-attempts: 10,
+ directory: {
+ # Override this property if custom file system should be used to create remote directories
+ # instead of default taken from Hadoop configuration
+ fs: "hdfs:///",
+ # Set this property if custom absolute root should be used for remote directories
+ root: "/app/drill"
+ }
+ }
}
# Below SSL parameters need to be set for custom transport layer settings.
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index bb451c9..0eb9c58 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -116,9 +116,9 @@ public interface ExecConstants {
* Configuration properties connected with dynamic UDFs support
*/
String UDF_RETRY_ATTEMPTS = "drill.exec.udf.retry-attempts";
+ String UDF_DIRECTORY_LOCAL = "drill.exec.udf.directory.local";
String UDF_DIRECTORY_FS = "drill.exec.udf.directory.fs";
String UDF_DIRECTORY_ROOT = "drill.exec.udf.directory.root";
- String UDF_DIRECTORY_LOCAL = "drill.exec.udf.directory.local";
String UDF_DIRECTORY_STAGING = "drill.exec.udf.directory.staging";
String UDF_DIRECTORY_REGISTRY = "drill.exec.udf.directory.registry";
String UDF_DIRECTORY_TMP = "drill.exec.udf.directory.tmp";
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index d0aa33f..590a99a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -377,7 +377,6 @@ public class FunctionImplementationRegistry implements FunctionLookupContext, Au
* Creates local udf directory, if it doesn't exist.
* Checks if local udf directory is a directory and if current application has write rights on it.
* Attempts to clean up local udf directory in case jars were left after previous drillbit run.
- * Local udf directory path is concatenated from drill temporary directory and ${drill.exec.udf.directory.local}.
*
* @param config drill config
* @return path to local udf directory
@@ -395,22 +394,24 @@ public class FunctionImplementationRegistry implements FunctionLookupContext, Au
} catch (IOException e) {
throw new DrillRuntimeException("Error during local udf directory clean up", e);
}
+ logger.info("Created and validated local udf directory [{}]", udfPath);
return new Path(udfDir.toURI());
}
/**
- * First tries to get drill temporary directory value from environmental variable $DRILL_TMP_DIR,
- * then from config ${drill.tmp-dir}.
+ * First tries to get drill temporary directory value from from config ${drill.tmp-dir},
+ * then checks environmental variable $DRILL_TMP_DIR.
* If value is still missing, generates directory using {@link Files#createTempDir()}.
* If temporary directory was generated, sets {@link #deleteTmpDir} to true
* to delete directory on drillbit exit.
* @return drill temporary directory path
*/
private File getTmpDir(DrillConfig config) {
- String drillTempDir = System.getenv("DRILL_TMP_DIR");
-
- if (drillTempDir == null && config.hasPath(ExecConstants.DRILL_TMP_DIR)) {
+ String drillTempDir;
+ if (config.hasPath(ExecConstants.DRILL_TMP_DIR)) {
drillTempDir = config.getString(ExecConstants.DRILL_TMP_DIR);
+ } else {
+ drillTempDir = System.getenv("DRILL_TMP_DIR");
}
if (drillTempDir == null) {
@@ -477,7 +478,7 @@ public class FunctionImplementationRegistry implements FunctionLookupContext, Au
* Will unregister all functions associated with the jar name
* and delete binary and source associated with the jar from local udf directory
*/
- public class UnregistrationListener implements TransientStoreListener {
+ private class UnregistrationListener implements TransientStoreListener {
@Override
public void onChange(TransientStoreEvent event) {
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
index 4ce4a19..e5e62eb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
@@ -81,8 +81,7 @@ import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT;
*/
public class RemoteFunctionRegistry implements AutoCloseable {
- public static final String REGISTRY = "registry";
-
+ private static final String registry_path = "registry";
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RemoteFunctionRegistry.class);
private static final ObjectMapper mapper = new ObjectMapper().enable(INDENT_OUTPUT);
@@ -108,15 +107,15 @@ public class RemoteFunctionRegistry implements AutoCloseable {
}
public Registry getRegistry() {
- return registry.get(REGISTRY, null);
+ return registry.get(registry_path, null);
}
public Registry getRegistry(DataChangeVersion version) {
- return registry.get(REGISTRY, version);
+ return registry.get(registry_path, version);
}
public void updateRegistry(Registry registryContent, DataChangeVersion version) throws VersionMismatchException {
- registry.put(REGISTRY, registryContent, version);
+ registry.put(registry_path, registryContent, version);
}
public void submitForUnregistration(String jar) {
@@ -168,7 +167,7 @@ public class RemoteFunctionRegistry implements AutoCloseable {
.persist()
.build();
registry = storeProvider.getOrCreateStore(registrationConfig);
- registry.putIfAbsent(REGISTRY, Registry.getDefaultInstance());
+ registry.putIfAbsent(registry_path, Registry.getDefaultInstance());
} catch (StoreException e) {
throw new DrillRuntimeException("Failure while loading remote registry.", e);
}
@@ -189,6 +188,7 @@ public class RemoteFunctionRegistry implements AutoCloseable {
* if not set, uses user home directory instead.
*/
private void prepareAreas(DrillConfig config) {
+ logger.info("Preparing three remote udf areas: staging, registry and tmp.");
Configuration conf = new Configuration();
if (config.hasPath(ExecConstants.UDF_DIRECTORY_FS)) {
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.getString(ExecConstants.UDF_DIRECTORY_FS));
@@ -245,6 +245,7 @@ public class RemoteFunctionRegistry implements AutoCloseable {
} catch (Exception e) {
DrillRuntimeException.format(e, "Error during udf area creation [%s] on file system [%s]", fullPath, fs.getUri());
}
+ logger.info("Created remote udf area [{}] on file system [{}]", fullPath, fs.getUri());
return path;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index deb31b3..ab73cec 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -45,11 +45,10 @@ drill.client: {
supports-complex-types: true
}
-// Directory is used as base for temporary storage of Dynamic UDF jars.
-// Set this property if you want to have custom temporary directory, instead of generated at runtime.
-// By default ${DRILL_TMP_DIR} is used if set.
-// drill.tmp-dir: "/tmp"
-// drill.tmp-dir: ${?DRILL_TMP_DIR}
+// Location Drill uses for temporary files, such as downloaded dynamic UDFs jars.
+// By default ${DRILL_TMP_DIR} is used if set or ${drill.tmp-dir} if it's been overridden.
+drill.tmp-dir: "/tmp"
+drill.tmp-dir: ${?DRILL_TMP_DIR}
drill.exec: {
cluster-id: "drillbits1"
@@ -202,24 +201,25 @@ drill.exec: {
udf: {
retry-attempts: 5,
directory: {
+ // Base directory for remote and local udf directories, unique among clusters.
+ base: ${drill.exec.zk.root}"/udf",
+
+ // Path to local udf directory, always created on local file system.
+ // Root for these directory is generated at runtime unless Drill temporary directory is set.
+ local: ${drill.exec.udf.directory.base}"/udf/local",
+
// Set this property if custom file system should be used to create remote directories, ex: fs: "file:///".
// fs: "",
// Set this property if custom absolute root should be used for remote directories, ex: root: "/app/drill".
// root: "",
- // Relative base directory for all udf directories (local and remote).
- base: ${drill.exec.zk.root}"/udf",
-
- // Relative path to local udf directory.
- // Generated temporary directory is used as root unless ${DRILL_TMP_DIR} or ${drill.tmp-dir} is set.
- local: ${drill.exec.udf.directory.base}"/local"
-
// Relative path to all remote udf directories.
- // Directories are created under default file system unless ${drill.exec.udf.directory.fs} is set.
+ // Directories are created under default file system taken from Hadoop configuration
+ // unless ${drill.exec.udf.directory.fs} is set.
// User home directory is used as root unless ${drill.exec.udf.directory.root} is set.
staging: ${drill.exec.udf.directory.base}"/staging",
registry: ${drill.exec.udf.directory.base}"/registry",
- tmp: ${drill.exec.udf.directory.base}"/tmp",
+ tmp: ${drill.exec.udf.directory.base}"/tmp"
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
index 4676e39..78cdbe2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
@@ -78,6 +78,7 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
public void setup() {
Properties overrideProps = new Properties();
overrideProps.setProperty("drill.exec.udf.directory.root", base.getRoot().getPath());
+ overrideProps.setProperty("drill.tmp-dir", base.getRoot().getPath());
updateTestCluster(1, DrillConfig.create(overrideProps));
}
[13/14] drill git commit: DRILL-4938: Report UserException when
constant expression reduction fails
Posted by su...@apache.org.
DRILL-4938: Report UserException when constant expression reduction fails
closes #689
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/cf2b7c70
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/cf2b7c70
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/cf2b7c70
Branch: refs/heads/master
Commit: cf2b7c70ea47e604364ccb5e8076bbd7eae310c3
Parents: 97da019
Author: Serhii-Harnyk <se...@gmail.com>
Authored: Fri Nov 18 16:02:04 2016 +0000
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:59:59 2016 -0800
----------------------------------------------------------------------
.../planner/logical/DrillConstExecutor.java | 13 +-
.../test/TestExecutionExceptionsToClient.java | 177 ++++++++++---------
2 files changed, 100 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/cf2b7c70/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
index 829db2a..4a94c71 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.planner.logical;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import io.netty.buffer.DrillBuf;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionStringBuilder;
import org.apache.drill.common.expression.LogicalExpression;
@@ -68,7 +68,6 @@ import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.calcite.util.NlsString;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.planner.sql.TypeInferenceUtils;
import org.joda.time.DateTime;
@@ -129,8 +128,9 @@ public class DrillConstExecutor implements RelOptPlanner.Executor {
String message = String.format(
"Failure while materializing expression in constant expression evaluator [%s]. Errors: %s",
newCall.toString(), errors.toString());
- logger.error(message);
- throw new DrillRuntimeException(message);
+ throw UserException.planError()
+ .message(message)
+ .build(logger);
}
if (NON_REDUCIBLE_TYPES.contains(materializedExpr.getMajorType().getMinorType())) {
@@ -149,8 +149,9 @@ public class DrillConstExecutor implements RelOptPlanner.Executor {
if (sqlTypeName == null) {
String message = String.format("Error reducing constant expression, unsupported type: %s.",
materializedExpr.getMajorType().getMinorType());
- logger.error(message);
- throw new DrillRuntimeException(message);
+ throw UserException.unsupportedError()
+ .message(message)
+ .build(logger);
}
reducedValues.add(rexBuilder.makeNullLiteral(sqlTypeName));
continue;
http://git-wip-us.apache.org/repos/asf/drill/blob/cf2b7c70/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestExecutionExceptionsToClient.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestExecutionExceptionsToClient.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestExecutionExceptionsToClient.java
index d845a38..749312f 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestExecutionExceptionsToClient.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestExecutionExceptionsToClient.java
@@ -17,26 +17,20 @@
*/
package org.apache.drill.jdbc.test;
-import static org.junit.Assert.fail;
import static org.junit.Assert.assertThat;
import static org.hamcrest.CoreMatchers.*;
-import org.junit.After;
import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.jdbc.Driver;
import org.apache.drill.jdbc.JdbcTestBase;
import java.sql.Connection;
-import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
-
public class TestExecutionExceptionsToClient extends JdbcTestBase {
private static Connection connection;
@@ -51,136 +45,151 @@ public class TestExecutionExceptionsToClient extends JdbcTestBase {
connection.close();
}
- @Test
+ @Test(expected = SQLException.class)
public void testExecuteQueryThrowsRight1() throws Exception {
final Statement statement = connection.createStatement();
try {
- statement.executeQuery( "SELECT one case of syntax error" );
- }
- catch ( SQLException e ) {
- assertThat( "Null getCause(); missing expected wrapped exception",
- e.getCause(), notNullValue() );
+ statement.executeQuery("SELECT one case of syntax error");
+ } catch (SQLException e) {
+ assertThat("Null getCause(); missing expected wrapped exception",
+ e.getCause(), notNullValue());
- assertThat( "Unexpectedly wrapped another SQLException",
- e.getCause(), not( instanceOf( SQLException.class ) ) );
+ assertThat("Unexpectedly wrapped another SQLException",
+ e.getCause(), not(instanceOf(SQLException.class)));
- assertThat( "getCause() not UserRemoteException as expected",
- e.getCause(), instanceOf( UserRemoteException.class ) );
+ assertThat("getCause() not UserRemoteException as expected",
+ e.getCause(), instanceOf(UserRemoteException.class));
- assertThat( "No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
- e.getMessage(), anyOf( startsWith( "SYSTEM ERROR" ),
- startsWith( "PARSE ERROR" ) ) );
+ assertThat("No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
+ e.getMessage(), anyOf(startsWith("SYSTEM ERROR"), startsWith("PARSE ERROR")));
+ throw e;
}
}
- @Test
+ @Test(expected = SQLException.class)
public void testExecuteThrowsRight1() throws Exception {
final Statement statement = connection.createStatement();
try {
- statement.execute( "SELECT one case of syntax error" );
- }
- catch ( SQLException e ) {
- assertThat( "Null getCause(); missing expected wrapped exception",
- e.getCause(), notNullValue() );
+ statement.execute("SELECT one case of syntax error");
+ } catch (SQLException e) {
+ assertThat("Null getCause(); missing expected wrapped exception",
+ e.getCause(), notNullValue());
- assertThat( "Unexpectedly wrapped another SQLException",
- e.getCause(), not( instanceOf( SQLException.class ) ) );
+ assertThat("Unexpectedly wrapped another SQLException",
+ e.getCause(), not(instanceOf(SQLException.class)));
- assertThat( "getCause() not UserRemoteException as expected",
- e.getCause(), instanceOf( UserRemoteException.class ) );
+ assertThat("getCause() not UserRemoteException as expected",
+ e.getCause(), instanceOf(UserRemoteException.class));
- assertThat( "No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
- e.getMessage(), anyOf( startsWith( "SYSTEM ERROR" ),
- startsWith( "PARSE ERROR" ) ) );
+ assertThat("No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
+ e.getMessage(), anyOf(startsWith("SYSTEM ERROR"), startsWith("PARSE ERROR")));
+ throw e;
}
}
- @Test
+ @Test(expected = SQLException.class)
public void testExecuteUpdateThrowsRight1() throws Exception {
final Statement statement = connection.createStatement();
try {
- statement.executeUpdate( "SELECT one case of syntax error" );
- }
- catch ( SQLException e ) {
- assertThat( "Null getCause(); missing expected wrapped exception",
- e.getCause(), notNullValue() );
+ statement.executeUpdate("SELECT one case of syntax error");
+ } catch (SQLException e) {
+ assertThat("Null getCause(); missing expected wrapped exception",
+ e.getCause(), notNullValue());
- assertThat( "Unexpectedly wrapped another SQLException",
- e.getCause(), not( instanceOf( SQLException.class ) ) );
+ assertThat("Unexpectedly wrapped another SQLException",
+ e.getCause(), not(instanceOf(SQLException.class)));
- assertThat( "getCause() not UserRemoteException as expected",
- e.getCause(), instanceOf( UserRemoteException.class ) );
+ assertThat("getCause() not UserRemoteException as expected",
+ e.getCause(), instanceOf(UserRemoteException.class));
- assertThat( "No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
- e.getMessage(), anyOf( startsWith( "SYSTEM ERROR" ),
- startsWith( "PARSE ERROR" ) ) );
+ assertThat("No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
+ e.getMessage(), anyOf(startsWith("SYSTEM ERROR"), startsWith("PARSE ERROR")));
+ throw e;
}
}
- @Test
+ @Test(expected = SQLException.class)
public void testExecuteQueryThrowsRight2() throws Exception {
final Statement statement = connection.createStatement();
try {
- statement.executeQuery( "BAD QUERY 1" );
- }
- catch ( SQLException e ) {
- assertThat( "Null getCause(); missing expected wrapped exception",
- e.getCause(), notNullValue() );
+ statement.executeQuery("BAD QUERY 1");
+ } catch (SQLException e) {
+ assertThat("Null getCause(); missing expected wrapped exception",
+ e.getCause(), notNullValue());
- assertThat( "Unexpectedly wrapped another SQLException",
- e.getCause(), not( instanceOf( SQLException.class ) ) );
+ assertThat("Unexpectedly wrapped another SQLException",
+ e.getCause(), not(instanceOf(SQLException.class)));
- assertThat( "getCause() not UserRemoteException as expected",
- e.getCause(), instanceOf( UserRemoteException.class ) );
+ assertThat("getCause() not UserRemoteException as expected",
+ e.getCause(), instanceOf(UserRemoteException.class));
- assertThat( "No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
- e.getMessage(), anyOf( startsWith( "SYSTEM ERROR" ),
- startsWith( "PARSE ERROR" ) ) );
+ assertThat("No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
+ e.getMessage(), anyOf(startsWith("SYSTEM ERROR"), startsWith("PARSE ERROR")));
+ throw e;
}
}
- @Test
+ @Test(expected = SQLException.class)
public void testExecuteThrowsRight2() throws Exception {
final Statement statement = connection.createStatement();
try {
- statement.execute( "worse query 2" );
- }
- catch ( SQLException e ) {
- assertThat( "Null getCause(); missing expected wrapped exception",
- e.getCause(), notNullValue() );
+ statement.execute("worse query 2");
+ } catch (SQLException e) {
+ assertThat("Null getCause(); missing expected wrapped exception",
+ e.getCause(), notNullValue());
- assertThat( "Unexpectedly wrapped another SQLException",
- e.getCause(), not( instanceOf( SQLException.class ) ) );
+ assertThat("Unexpectedly wrapped another SQLException",
+ e.getCause(), not(instanceOf(SQLException.class)));
- assertThat( "getCause() not UserRemoteException as expected",
- e.getCause(), instanceOf( UserRemoteException.class ) );
+ assertThat("getCause() not UserRemoteException as expected",
+ e.getCause(), instanceOf(UserRemoteException.class));
- assertThat( "No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
- e.getMessage(), anyOf( startsWith( "SYSTEM ERROR" ),
- startsWith( "PARSE ERROR" ) ) );
+ assertThat("No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
+ e.getMessage(), anyOf(startsWith("SYSTEM ERROR"), startsWith("PARSE ERROR")));
+ throw e;
}
}
- @Test
+ @Test(expected = SQLException.class)
public void testExecuteUpdateThrowsRight2() throws Exception {
final Statement statement = connection.createStatement();
try {
- statement.executeUpdate( "naughty, naughty query 3" );
- }
- catch ( SQLException e ) {
- assertThat( "Null getCause(); missing expected wrapped exception",
- e.getCause(), notNullValue() );
+ statement.executeUpdate("naughty, naughty query 3");
+ } catch (SQLException e) {
+ assertThat("Null getCause(); missing expected wrapped exception",
+ e.getCause(), notNullValue());
- assertThat( "Unexpectedly wrapped another SQLException",
- e.getCause(), not( instanceOf( SQLException.class ) ) );
+ assertThat("Unexpectedly wrapped another SQLException",
+ e.getCause(), not(instanceOf(SQLException.class)));
- assertThat( "getCause() not UserRemoteException as expected",
- e.getCause(), instanceOf( UserRemoteException.class ) );
+ assertThat("getCause() not UserRemoteException as expected",
+ e.getCause(), instanceOf(UserRemoteException.class));
- assertThat( "No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
- e.getMessage(), anyOf( startsWith( "SYSTEM ERROR" ),
- startsWith( "PARSE ERROR" ) ) );
+ assertThat("No expected current \"SYSTEM ERROR\"/eventual \"PARSE ERROR\"",
+ e.getMessage(), anyOf(startsWith("SYSTEM ERROR"), startsWith("PARSE ERROR")));
+ throw e;
}
}
+ @Test(expected = SQLException.class)
+ public void testMaterializingError() throws Exception {
+ final Statement statement = connection.createStatement();
+ try {
+ statement.executeUpdate("select (res1 = 2016/09/22) res2 from (select (case when (false) then null else "
+ + "cast('2016/09/22' as date) end) res1 from (values(1)) foo) foobar");
+ } catch (SQLException e) {
+ assertThat("Null getCause(); missing expected wrapped exception",
+ e.getCause(), notNullValue());
+
+ assertThat("Unexpectedly wrapped another SQLException",
+ e.getCause(), not(instanceOf(SQLException.class)));
+
+ assertThat("getCause() not UserRemoteException as expected",
+ e.getCause(), instanceOf(UserRemoteException.class));
+
+ assertThat("No expected current \"PLAN ERROR\"",
+ e.getMessage(), startsWith("PLAN ERROR"));
+ throw e;
+ }
+ }
}
[04/14] drill git commit: DRILL-5112: Fix config in PopUnitTestBase
Posted by su...@apache.org.
DRILL-5112: Fix config in PopUnitTestBase
Tests rely on command-line settings in the pom.xml file. Those
settings are not available when tests are run in Eclipse.
Replicated required settings into the base test class (as in
BaseTestQuery).
closes #681
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/963955b9
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/963955b9
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/963955b9
Branch: refs/heads/master
Commit: 963955b94ab75ce52894582791f6669d578af562
Parents: fb32bfe
Author: Paul Rogers <pr...@maprtech.com>
Authored: Tue Dec 6 21:11:56 2016 -0800
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:02 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/drill/exec/server/Drillbit.java | 2 +-
.../impl/mergereceiver/TestMergingReceiver.java | 3 +--
.../org/apache/drill/exec/pop/PopUnitTestBase.java | 16 +++++++++++++++-
3 files changed, 17 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/963955b9/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
index a28dc91..547915e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
@@ -64,7 +64,7 @@ public class Drillbit implements AutoCloseable {
Environment.logEnv("Drillbit environment: ", logger);
}
- private final static String SYSTEM_OPTIONS_NAME = "org.apache.drill.exec.server.Drillbit.system_options";
+ public final static String SYSTEM_OPTIONS_NAME = "org.apache.drill.exec.server.Drillbit.system_options";
private boolean isClosed = false;
http://git-wip-us.apache.org/repos/asf/drill/blob/963955b9/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
index 537a583..71a5070 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
@@ -37,7 +37,6 @@ import org.apache.drill.exec.vector.ValueVector;
import org.junit.Test;
import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
import com.google.common.io.Files;
public class TestMergingReceiver extends PopUnitTestBase {
@@ -95,7 +94,7 @@ public class TestMergingReceiver extends PopUnitTestBase {
final int batchRowCount = queryData.getRowCount();
count += batchRowCount;
batchLoader.load(queryData.getDef(), b.getData());
- for (final VectorWrapper vw : batchLoader) {
+ for (final VectorWrapper<?> vw : batchLoader) {
final ValueVector vv = vw.getValueVector();
final ValueVector.Accessor va = vv.getAccessor();
final MaterializedField materializedField = vv.getField();
http://git-wip-us.apache.org/repos/asf/drill/blob/963955b9/exec/java-exec/src/test/java/org/apache/drill/exec/pop/PopUnitTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/PopUnitTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/PopUnitTestBase.java
index 56543ca..b76aad3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/PopUnitTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/PopUnitTestBase.java
@@ -18,9 +18,12 @@
package org.apache.drill.exec.pop;
import java.io.IOException;
+import java.util.Properties;
+import org.apache.drill.QueryTestUtil;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.exception.FragmentSetupException;
import org.apache.drill.exec.physical.PhysicalPlan;
@@ -29,6 +32,7 @@ import org.apache.drill.exec.planner.PhysicalPlanReader;
import org.apache.drill.exec.planner.fragment.Fragment;
import org.apache.drill.exec.planner.fragment.Fragment.ExchangeFragmentPair;
import org.apache.drill.exec.planner.fragment.MakeFragmentsVisitor;
+import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.work.foreman.ForemanSetupException;
import org.junit.BeforeClass;
@@ -42,7 +46,17 @@ public abstract class PopUnitTestBase extends ExecTest{
@BeforeClass
public static void setup() {
- CONFIG = DrillConfig.create();
+ Properties props = new Properties();
+
+ // Properties here mimic those in drill-root/pom.xml, Surefire plugin
+ // configuration. They allow tests to run successfully in Eclipse.
+
+ props.put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, "false");
+ props.put(ExecConstants.HTTP_ENABLE, "false");
+ props.put(Drillbit.SYSTEM_OPTIONS_NAME, "org.apache.drill.exec.compile.ClassTransformer.scalar_replacement=on");
+ props.put(QueryTestUtil.TEST_QUERY_PRINTING_SILENT, "true");
+ props.put("drill.catastrophic_to_standard_out", "true");
+ CONFIG = DrillConfig.create(props);
}
[08/14] drill git commit: DRILL-5056: Fix UserException to write full
message to log
Posted by su...@apache.org.
DRILL-5056: Fix UserException to write full message to log
A case occurred in which the External Sort failed during spilling.
All that was written to the log was:
2016-11-18 ... INFO o.a.d.e.p.i.xsort.ExternalSortBatch - User Error Occurred
Modified the logging code to provide more information to aid tracking down problems that occur in the field.
closes #665
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/641d35de
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/641d35de
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/641d35de
Branch: refs/heads/master
Commit: 641d35de03e84361ffe038fe30b717012c1571b8
Parents: 963955b
Author: Paul Rogers <pr...@maprtech.com>
Authored: Mon Nov 21 10:19:15 2016 -0800
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:03 2016 -0800
----------------------------------------------------------------------
.../apache/drill/common/exceptions/UserException.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/641d35de/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/UserException.java b/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
index 35e71d1..87b3fd4 100644
--- a/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
+++ b/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
@@ -38,6 +38,7 @@ import org.slf4j.Logger;
* @see org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType
*/
public class UserException extends DrillRuntimeException {
+ private static final long serialVersionUID = -6720929331624621840L;
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UserException.class);
public static final String MEMORY_ERROR_MSG = "One or more nodes ran out of memory while executing the query.";
@@ -549,7 +550,15 @@ public class UserException extends DrillRuntimeException {
if (isSystemError) {
logger.error(newException.getMessage(), newException);
} else {
- logger.info("User Error Occurred", newException);
+ StringBuilder buf = new StringBuilder();
+ buf.append("User Error Occurred");
+ if (message != null) {
+ buf.append(": ").append(message);
+ }
+ if (cause != null) {
+ buf.append(" (").append(cause.getMessage()).append(")");
+ }
+ logger.info(buf.toString(), newException);
}
return newException;
[02/14] drill git commit: DRILL-5113: Upgrade Maven RAT plugin to
avoid annoying XML errors
Posted by su...@apache.org.
DRILL-5113: Upgrade Maven RAT plugin to avoid annoying XML errors
Upgrade to eliminate XML compiler warnings that appear for each
sub-project in the Drill build.
Also fixes a Maven warning about a duplicated version error for
the dependency plugin (version is inherited from root pom.xml).
closes #682
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/d9077a02
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/d9077a02
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/d9077a02
Branch: refs/heads/master
Commit: d9077a02d36c1d32c5310fb1728170b0f6f6995c
Parents: 11645dc
Author: Paul Rogers <pr...@maprtech.com>
Authored: Tue Dec 6 21:26:58 2016 -0800
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:01 2016 -0800
----------------------------------------------------------------------
exec/java-exec/pom.xml | 45 ++++++++++++++++++++++-----------------------
pom.xml | 1 +
2 files changed, 23 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/d9077a02/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index b54eaeb..5992acb 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -1,13 +1,13 @@
<?xml version="1.0"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
- license agreements. See the NOTICE file distributed with this work for additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
language governing permissions and limitations under the License. -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -26,7 +26,7 @@
<version>1.3</version>
<scope>test</scope>
</dependency>
-
+
<!-- <dependency> -->
<!-- <groupId>org.ow2.asm</groupId> -->
<!-- <artifactId>asm-util</artifactId> -->
@@ -54,13 +54,13 @@
<artifactId>commons-pool2</artifactId>
<version>2.1</version>
</dependency>
-
+
<dependency>
<groupId>com.univocity</groupId>
<artifactId>univocity-parsers</artifactId>
<version>1.3.0</version>
</dependency>
-
+
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math</artifactId>
@@ -71,7 +71,7 @@
<artifactId>paranamer</artifactId>
<version>2.5.6</version>
<exclusions>
- <!-- Exclude Paranamer's dependency on mockito-all, which has a Jar
+ <!-- Exclude Paranamer's dependency on mockito-all, which has a Jar
file that contains old Hamcrest 1.1 classes (rather than declaring
a dependency on Hamcrest). (See DRILL-2130.) -->
<exclusion>
@@ -321,7 +321,7 @@
<groupId>org.apache.drill</groupId>
<artifactId>drill-logical</artifactId>
<version>${project.version}</version>
- </dependency>
+ </dependency>
<dependency>
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-rpc</artifactId>
@@ -518,7 +518,7 @@
</profiles>
<build>
-
+
<plugins>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
@@ -542,9 +542,8 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
<executions>
- <!-- Extract parser grammar template from calcite-core.jar and put
+ <!-- Extract parser grammar template from calcite-core.jar and put
it under ${project.build.directory}/codegen/templates where all freemarker templates are. -->
<execution>
<id>unpack-parser-template</id>
@@ -565,8 +564,8 @@
</artifactItems>
</configuration>
</execution>
-
- <!-- Extract ValueVectorTypes.tdd from drill-vector.jar and put
+
+ <!-- Extract ValueVectorTypes.tdd from drill-vector.jar and put
it under ${project.build.directory}/codegen/data where all freemarker data files are. -->
<execution>
<id>unpack-vector-types</id>
@@ -586,7 +585,7 @@
</artifactItem>
</artifactItems>
</configuration>
- </execution>
+ </execution>
</executions>
</plugin>
@@ -702,8 +701,8 @@
<artifactId>maven-surefire-plugin</artifactId>
<executions>
<execution>
- <!-- we override the default test execution to exclude tests
- that would take unusually long time to run (> 1 minute) and run such tests
+ <!-- we override the default test execution to exclude tests
+ that would take unusually long time to run (> 1 minute) and run such tests
in a different profile -->
<id>default-test</id>
<phase>test</phase>
@@ -719,7 +718,7 @@
</plugins>
<pluginManagement>
<plugins>
- <!--This plugin's configuration is used to store Eclipse m2e settings
+ <!--This plugin's configuration is used to store Eclipse m2e settings
only. It has no influence on the Maven build itself. -->
<plugin>
<groupId>org.eclipse.m2e</groupId>
http://git-wip-us.apache.org/repos/asf/drill/blob/d9077a02/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 856e332..6772551 100644
--- a/pom.xml
+++ b/pom.xml
@@ -176,6 +176,7 @@
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
+ <version>0.12</version>
<executions>
<execution>
<id>rat-checks</id>
[14/14] drill git commit: DRILL-5065: Optimize count(*) queries on
MapR-DB JSON Tables
Posted by su...@apache.org.
DRILL-5065: Optimize count(*) queries on MapR-DB JSON Tables
In MapR-DB v5.2.0, we enabled '_id' only projection for JSON
tables. Hence, we can now optimize the following queries:
a. count(*) by projecting only the '_id' column.
b. '_id' only projections, including count(_id)
Change the format plugin config parameter name.
Fix setter of config parameter `disableCountOptimization` for drill-maprdb plugin
closes #678
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/97da0199
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/97da0199
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/97da0199
Branch: refs/heads/master
Commit: 97da01995ecfd9a7d157ad183f0656cd15993f26
Parents: 94f3ba0
Author: Smidth Panchamia <sp...@mapr.com>
Authored: Mon Nov 28 13:59:34 2016 -0800
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:59:59 2016 -0800
----------------------------------------------------------------------
.../store/mapr/db/MapRDBFormatPluginConfig.java | 12 ++++
.../mapr/db/json/MaprDBJsonRecordReader.java | 64 ++++++++++++--------
2 files changed, 51 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/97da0199/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java
index 1bb07ed..8b89b78 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java
@@ -31,6 +31,7 @@ public class MapRDBFormatPluginConfig extends TableFormatPluginConfig {
public boolean enablePushdown = true;
public boolean ignoreSchemaChange = false;
public boolean readAllNumbersAsDouble = false;
+ public boolean disableCountOptimization = false;
@Override
public int hashCode() {
@@ -48,6 +49,8 @@ public class MapRDBFormatPluginConfig extends TableFormatPluginConfig {
return false;
} else if (enablePushdown != other.enablePushdown) {
return false;
+ } else if (disableCountOptimization != other.disableCountOptimization) {
+ return false;
}
return true;
}
@@ -65,6 +68,15 @@ public class MapRDBFormatPluginConfig extends TableFormatPluginConfig {
allTextMode = mode;
}
+ @JsonProperty("disableCountOptimization")
+ public void setDisableCountOptimization(boolean mode) {
+ disableCountOptimization = mode;
+ }
+
+ public boolean shouldDisableCountOptimization() {
+ return disableCountOptimization;
+ }
+
@JsonProperty("readAllNumbersAsDouble")
public void setReadAllNumbersAsDouble(boolean read) {
readAllNumbersAsDouble = read;
http://git-wip-us.apache.org/repos/asf/drill/blob/97da0199/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java
index ee6b15d..3105bec 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.store.mapr.db.json;
import static org.ojai.DocumentConstants.ID_KEY;
+import static org.ojai.DocumentConstants.ID_FIELD;
import java.nio.ByteBuffer;
import java.util.Collection;
@@ -93,6 +94,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
private boolean disablePushdown;
private final boolean allTextMode;
private final boolean ignoreSchemaChange;
+ private final boolean disableCountOptimization;
public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec,
MapRDBFormatPluginConfig formatPluginConfig,
@@ -110,6 +112,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
condition = com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(serializedFilter));
}
+ disableCountOptimization = formatPluginConfig.shouldDisableCountOptimization();
setColumns(projectedColumns);
unionEnabled = context.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE);
readNumbersAsDouble = formatPluginConfig.isReadAllNumbersAsDouble();
@@ -121,36 +124,47 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
@Override
protected Collection<SchemaPath> transformColumns(Collection<SchemaPath> columns) {
Set<SchemaPath> transformed = Sets.newLinkedHashSet();
- if (!isStarQuery() && !disablePushdown) {
- Set<FieldPath> projectedFieldsSet = Sets.newTreeSet();
- for (SchemaPath column : columns) {
- if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) {
- /*
- * we do not include _id field in the set of projected fields
- * because the DB currently can not return a document if only
- * the _id field was projected. This should really be fixed in
- * the DB client (Bug 21708) to avoid transferring the entire
- * document when only _id is requested.
- */
- // projectedFieldsList.add(ID_FIELD);
- includeId = true;
- } else {
- projectedFieldsSet.add(getFieldPathForProjection(column));
+ if (disablePushdown) {
+ transformed.add(AbstractRecordReader.STAR_COLUMN);
+ includeId = true;
+ return transformed;
+ }
+
+ if (isStarQuery()) {
+ transformed.add(AbstractRecordReader.STAR_COLUMN);
+ includeId = true;
+ if (isSkipQuery()) {
+ // `SELECT COUNT(*)` query
+ if (!disableCountOptimization) {
+ projectedFields = new FieldPath[1];
+ projectedFields[0] = ID_FIELD;
}
- transformed.add(column);
}
- if (projectedFieldsSet.size() > 0) {
- projectedFields = projectedFieldsSet.toArray(new FieldPath[projectedFieldsSet.size()]);
+ return transformed;
+ }
+
+ Set<FieldPath> projectedFieldsSet = Sets.newTreeSet();
+ for (SchemaPath column : columns) {
+ if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) {
+ includeId = true;
+ if (!disableCountOptimization) {
+ projectedFieldsSet.add(ID_FIELD);
+ }
+ } else {
+ projectedFieldsSet.add(getFieldPathForProjection(column));
}
- } else {
- transformed.add(AbstractRecordReader.STAR_COLUMN);
- includeId = true;
+
+ transformed.add(column);
+ }
+
+ if (projectedFieldsSet.size() > 0) {
+ projectedFields = projectedFieldsSet.toArray(new FieldPath[projectedFieldsSet.size()]);
+ }
+
+ if (disableCountOptimization) {
+ idOnly = (projectedFields == null);
}
- /*
- * (Bug 21708) if we are projecting only the id field, save that condition here.
- */
- idOnly = !isStarQuery() && (projectedFields == null);
return transformed;
}
[10/14] drill git commit: DRILL-4812: Fix
FileSelection#handleWildCard to use normalized path separator
Posted by su...@apache.org.
DRILL-4812: Fix FileSelection#handleWildCard to use normalized path separator
closes #627
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b6561289
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b6561289
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b6561289
Branch: refs/heads/master
Commit: b6561289998da6023efa864212033fbf5ac84cbd
Parents: 8915611
Author: Mike Lavender <Mike Lavender>
Authored: Fri Oct 21 14:05:43 2016 -0700
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:03 2016 -0800
----------------------------------------------------------------------
.../main/java/org/apache/drill/exec/store/dfs/FileSelection.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/b6561289/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
index 3ab6277..e0f5438 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
@@ -369,7 +369,7 @@ public class FileSelection {
private static Path handleWildCard(final String root) {
if (root.contains(WILD_CARD)) {
int idx = root.indexOf(WILD_CARD); // first wild card in the path
- idx = root.lastIndexOf(PATH_SEPARATOR, idx); // file separator right before the first wild card
+ idx = root.lastIndexOf('/', idx); // file separator right before the first wild card
final String newRoot = root.substring(0, idx);
return new Path(newRoot);
} else {
[09/14] drill git commit: DRILL-5119: Update MapR version to
5.2.0.40963-mapr
Posted by su...@apache.org.
DRILL-5119: Update MapR version to 5.2.0.40963-mapr
closes #688
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/8915611e
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/8915611e
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/8915611e
Branch: refs/heads/master
Commit: 8915611e5a67745f89d4d06f0effbfe808035b75
Parents: 641d35d
Author: Patrick Wong <pw...@maprtech.com>
Authored: Fri Dec 9 14:26:31 2016 -0800
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:03 2016 -0800
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/8915611e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6772551..6813a72 100644
--- a/pom.xml
+++ b/pom.xml
@@ -38,7 +38,7 @@
<calcite.version>1.4.0-drill-r19</calcite.version>
<sqlline.version>1.1.9-drill-r7</sqlline.version>
<jackson.version>2.7.1</jackson.version>
- <mapr.release.version>5.2.0-mapr</mapr.release.version>
+ <mapr.release.version>5.2.0.40963-mapr</mapr.release.version>
<ojai.version>1.1</ojai.version>
<!--
[03/14] drill git commit: DRILL-5108: Reduce output from Maven
git-commit-id-plugin
Posted by su...@apache.org.
DRILL-5108: Reduce output from Maven git-commit-id-plugin
The git-commit-id-plugin grabs information from Git to display
during a build. It prints many e-mail addresses and other generic
project information. As part of the effort to trim down unit test
output, we propose to turn off the verbose output from this plugin
by default.
closes #680
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/11645dc8
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/11645dc8
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/11645dc8
Branch: refs/heads/master
Commit: 11645dc8c5439e40a864cef085e102da5e25401d
Parents: 351dea6
Author: Paul Rogers <pr...@maprtech.com>
Authored: Tue Dec 6 19:38:38 2016 -0800
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:01 2016 -0800
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/11645dc8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 18ae779..856e332 100644
--- a/pom.xml
+++ b/pom.xml
@@ -383,7 +383,7 @@
<configuration>
<dateFormat>dd.MM.yyyy '@' HH:mm:ss z</dateFormat>
- <verbose>true</verbose>
+ <verbose>false</verbose>
<skipPoms>false</skipPoms>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<failOnNoGitDirectory>false</failOnNoGitDirectory>
[05/14] drill git commit: DRILL-5044: Fix retry logic to handle
VersionMismatchException by not deleting jars in remote UDFs area
Posted by su...@apache.org.
DRILL-5044: Fix retry logic to handle VersionMismatchException by not deleting jars in remote UDFs area
closes #669
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/fb32bfe2
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/fb32bfe2
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/fb32bfe2
Branch: refs/heads/master
Commit: fb32bfe209b03ff269313aed70bf066b29314327
Parents: 85a9651
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Fri Nov 25 16:44:08 2016 +0000
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:02 2016 -0800
----------------------------------------------------------------------
.../sql/handlers/CreateFunctionHandler.java | 54 ++++++-----
.../sql/handlers/DropFunctionHandler.java | 47 +++++-----
.../org/apache/drill/TestDynamicUDFSupport.java | 96 +++++++++++++++++++-
3 files changed, 146 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/fb32bfe2/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateFunctionHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateFunctionHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateFunctionHandler.java
index 8515c8a..48bfd8b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateFunctionHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateFunctionHandler.java
@@ -90,7 +90,7 @@ public class CreateFunctionHandler extends DefaultSqlHandler {
jarManager.initRemoteBackup();
List<String> functions = validateAgainstLocalRegistry(jarManager, context.getFunctionRegistry());
- initRemoteRegistration(functions, jarManager, remoteRegistry, remoteRegistry.getRetryAttempts());
+ initRemoteRegistration(functions, jarManager, remoteRegistry);
jarManager.deleteQuietlyFromStagingArea();
return DirectPlan.createDirectPlan(context, true,
@@ -154,43 +154,47 @@ public class CreateFunctionHandler extends DefaultSqlHandler {
* Instantiates remote registration. First gets remote function registry with version.
* Version is used to ensure that we update the same registry we validated against.
* Then validates against list of remote jars.
- * If validation is successful, starts updating remote function registry.
+ * If validation is successful, first copies jars to registry area and starts updating remote function registry.
* If during update {@link VersionMismatchException} was detected,
- * calls itself recursively to instantiate new remote registration process.
- * Since remote registry version has changed, we need to re-validate against remote function registry one more time.
- * Each time recursive call occurs, decreases retry attempts counter by one.
+ * attempts to repeat remote registration process till retry attempts exceeds the limit.
* If retry attempts number hits 0, throws exception that failed to update remote function registry.
+ * In case of any error, if jars have been already copied to registry area, they will be deleted.
*
* @param functions list of functions present in jar
* @param jarManager helper class for copying jars to registry area
* @param remoteRegistry remote function registry
- * @param retryAttempts number of retry attempts
* @throws IOException in case of problems with copying jars to registry area
*/
private void initRemoteRegistration(List<String> functions,
- JarManager jarManager,
- RemoteFunctionRegistry remoteRegistry,
- int retryAttempts) throws IOException {
- DataChangeVersion version = new DataChangeVersion();
- List<Jar> remoteJars = remoteRegistry.getRegistry(version).getJarList();
- validateAgainstRemoteRegistry(remoteJars, jarManager.getBinaryName(), functions);
- jarManager.copyToRegistryArea();
- boolean cleanUp = true;
- List<Jar> jars = Lists.newArrayList(remoteJars);
- jars.add(Jar.newBuilder().setName(jarManager.getBinaryName()).addAllFunctionSignature(functions).build());
- Registry updatedRegistry = Registry.newBuilder().addAllJar(jars).build();
+ JarManager jarManager,
+ RemoteFunctionRegistry remoteRegistry) throws IOException {
+ int retryAttempts = remoteRegistry.getRetryAttempts();
+ boolean copyJars = true;
try {
- remoteRegistry.updateRegistry(updatedRegistry, version);
- cleanUp = false;
- } catch (VersionMismatchException ex) {
- if (retryAttempts-- == 0) {
- throw new DrillRuntimeException("Failed to update remote function registry. Exceeded retry attempts limit.");
+ while (retryAttempts >= 0) {
+ DataChangeVersion version = new DataChangeVersion();
+ List<Jar> remoteJars = remoteRegistry.getRegistry(version).getJarList();
+ validateAgainstRemoteRegistry(remoteJars, jarManager.getBinaryName(), functions);
+ if (copyJars) {
+ jarManager.copyToRegistryArea();
+ copyJars = false;
+ }
+ List<Jar> jars = Lists.newArrayList(remoteJars);
+ jars.add(Jar.newBuilder().setName(jarManager.getBinaryName()).addAllFunctionSignature(functions).build());
+ Registry updatedRegistry = Registry.newBuilder().addAllJar(jars).build();
+ try {
+ remoteRegistry.updateRegistry(updatedRegistry, version);
+ return;
+ } catch (VersionMismatchException ex) {
+ retryAttempts--;
+ }
}
- initRemoteRegistration(functions, jarManager, remoteRegistry, retryAttempts);
- } finally {
- if (cleanUp) {
+ throw new DrillRuntimeException("Failed to update remote function registry. Exceeded retry attempts limit.");
+ } catch (Exception e) {
+ if (!copyJars) {
jarManager.deleteQuietlyFromRegistryArea();
}
+ throw e;
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/fb32bfe2/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DropFunctionHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DropFunctionHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DropFunctionHandler.java
index 5269a4b..6e2801a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DropFunctionHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DropFunctionHandler.java
@@ -84,7 +84,7 @@ public class DropFunctionHandler extends DefaultSqlHandler {
return DirectPlan.createDirectPlan(context, false, String.format("Jar with %s name is used. Action: %s", jarName, action));
}
- Jar deletedJar = unregister(jarName, remoteFunctionRegistry, remoteFunctionRegistry.getRetryAttempts());
+ Jar deletedJar = unregister(jarName, remoteFunctionRegistry);
if (deletedJar == null) {
return DirectPlan.createDirectPlan(context, false, String.format("Jar %s is not registered in remote registry", jarName));
}
@@ -108,45 +108,44 @@ public class DropFunctionHandler extends DefaultSqlHandler {
}
/**
- * First gets remote function registry with version.
+ * Gets remote function registry with version.
* Version is used to ensure that we update the same registry we removed jars from.
* Looks for a jar to be deleted, if founds one,
- * attempts to update remote registry with updated list of jars, that excludes jar to be deleted.
+ * attempts to update remote registry with list of jars, that excludes jar to be deleted.
* If during update {@link VersionMismatchException} was detected,
- * calls itself recursively to instantiate new remote unregistration process.
- * Since remote registry version has changed we need to look for jar to be deleted one more time.
- * Each time recursive call occurs, decreases retry attempts counter by one.
+ * attempts to repeat unregistration process till retry attempts exceeds the limit.
* If retry attempts number hits 0, throws exception that failed to update remote function registry.
*
* @param jarName jar name
* @param remoteFunctionRegistry remote function registry
- * @param retryAttempts number of retry attempts
* @return jar that was unregistered, null otherwise
*/
- private Jar unregister(String jarName, RemoteFunctionRegistry remoteFunctionRegistry, int retryAttempts) {
- DataChangeVersion version = new DataChangeVersion();
- Registry registry = remoteFunctionRegistry.getRegistry(version);
- Jar jarToBeDeleted = null;
- List<Jar> jars = Lists.newArrayList();
- for (Jar j : registry.getJarList()) {
- if (j.getName().equals(jarName)) {
- jarToBeDeleted = j;
- } else {
- jars.add(j);
+ private Jar unregister(String jarName, RemoteFunctionRegistry remoteFunctionRegistry) {
+ int retryAttempts = remoteFunctionRegistry.getRetryAttempts();
+ while (retryAttempts >= 0) {
+ DataChangeVersion version = new DataChangeVersion();
+ Registry registry = remoteFunctionRegistry.getRegistry(version);
+ Jar jarToBeDeleted = null;
+ List<Jar> jars = Lists.newArrayList();
+ for (Jar j : registry.getJarList()) {
+ if (j.getName().equals(jarName)) {
+ jarToBeDeleted = j;
+ } else {
+ jars.add(j);
+ }
+ }
+ if (jarToBeDeleted == null) {
+ return null;
}
- }
- if (jarToBeDeleted != null) {
Registry updatedRegistry = Registry.newBuilder().addAllJar(jars).build();
try {
remoteFunctionRegistry.updateRegistry(updatedRegistry, version);
+ return jarToBeDeleted;
} catch (VersionMismatchException ex) {
- if (retryAttempts-- == 0) {
- throw new DrillRuntimeException("Failed to update remote function registry. Exceeded retry attempts limit.");
- }
- unregister(jarName, remoteFunctionRegistry, retryAttempts);
+ retryAttempts--;
}
}
- return jarToBeDeleted;
+ throw new DrillRuntimeException("Failed to update remote function registry. Exceeded retry attempts limit.");
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/fb32bfe2/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
index 78cdbe2..10a03b7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
@@ -177,7 +177,7 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
}
@Test
- public void testSuccessfulCreate() throws Exception {
+ public void testSuccessfulRegistration() throws Exception {
copyDefaultJarsToStagingArea();
String summary = "The following UDFs in jar %s have been registered:\n" +
@@ -272,6 +272,75 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
}
@Test
+ public void testSuccessfulRegistrationAfterSeveralRetryAttempts() throws Exception {
+ RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
+ copyDefaultJarsToStagingArea();
+
+ doThrow(new VersionMismatchException("Version mismatch detected", 1))
+ .doThrow(new VersionMismatchException("Version mismatch detected", 1))
+ .doCallRealMethod()
+ .when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
+
+ String summary = "The following UDFs in jar %s have been registered:\n" +
+ "[custom_lower(VARCHAR-REQUIRED)]";
+
+ testBuilder()
+ .sqlQuery("create function using jar '%s'", default_binary_name)
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format(summary, default_binary_name))
+ .go();
+
+ verify(remoteFunctionRegistry, times(3))
+ .updateRegistry(any(Registry.class), any(DataChangeVersion.class));
+
+ FileSystem fs = remoteFunctionRegistry.getFs();
+
+ assertFalse("Staging area should be empty", fs.listFiles(remoteFunctionRegistry.getStagingArea(), false).hasNext());
+ assertFalse("Temporary area should be empty", fs.listFiles(remoteFunctionRegistry.getTmpArea(), false).hasNext());
+
+ assertTrue("Binary should be present in registry area",
+ fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
+ assertTrue("Source should be present in registry area",
+ fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
+
+ Registry registry = remoteFunctionRegistry.getRegistry();
+ assertEquals("Registry should contain one jar", registry.getJarList().size(), 1);
+ assertEquals(registry.getJar(0).getName(), default_binary_name);
+ }
+
+ @Test
+ public void testSuccessfulUnregistrationAfterSeveralRetryAttempts() throws Exception {
+ RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
+ copyDefaultJarsToStagingArea();
+ test("create function using jar '%s'", default_binary_name);
+
+ reset(remoteFunctionRegistry);
+ doThrow(new VersionMismatchException("Version mismatch detected", 1))
+ .doThrow(new VersionMismatchException("Version mismatch detected", 1))
+ .doCallRealMethod()
+ .when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
+
+ String summary = "The following UDFs in jar %s have been unregistered:\n" +
+ "[custom_lower(VARCHAR-REQUIRED)]";
+
+ testBuilder()
+ .sqlQuery("drop function using jar '%s'", default_binary_name)
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format(summary, default_binary_name))
+ .go();
+
+ verify(remoteFunctionRegistry, times(3))
+ .updateRegistry(any(Registry.class), any(DataChangeVersion.class));
+
+ FileSystem fs = remoteFunctionRegistry.getFs();
+
+ assertFalse("Registry area should be empty", fs.listFiles(remoteFunctionRegistry.getRegistryArea(), false).hasNext());
+ assertEquals("Registry should be empty", remoteFunctionRegistry.getRegistry().getJarList().size(), 0);
+ }
+
+ @Test
public void testExceedRetryAttemptsDuringRegistration() throws Exception {
RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
copyDefaultJarsToStagingArea();
@@ -290,6 +359,18 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
verify(remoteFunctionRegistry, times(remoteFunctionRegistry.getRetryAttempts() + 1))
.updateRegistry(any(Registry.class), any(DataChangeVersion.class));
+
+ FileSystem fs = remoteFunctionRegistry.getFs();
+
+ assertTrue("Binary should be present in staging area",
+ fs.exists(new Path(remoteFunctionRegistry.getStagingArea(), default_binary_name)));
+ assertTrue("Source should be present in staging area",
+ fs.exists(new Path(remoteFunctionRegistry.getStagingArea(), default_source_name)));
+
+ assertFalse("Registry area should be empty", fs.listFiles(remoteFunctionRegistry.getRegistryArea(), false).hasNext());
+ assertFalse("Temporary area should be empty", fs.listFiles(remoteFunctionRegistry.getTmpArea(), false).hasNext());
+
+ assertEquals("Registry should be empty", remoteFunctionRegistry.getRegistry().getJarList().size(), 0);
}
@Test
@@ -313,6 +394,17 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
verify(remoteFunctionRegistry, times(remoteFunctionRegistry.getRetryAttempts() + 1))
.updateRegistry(any(Registry.class), any(DataChangeVersion.class));
+
+ FileSystem fs = remoteFunctionRegistry.getFs();
+
+ assertTrue("Binary should be present in registry area",
+ fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
+ assertTrue("Source should be present in registry area",
+ fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
+
+ Registry registry = remoteFunctionRegistry.getRegistry();
+ assertEquals("Registry should contain one jar", registry.getJarList().size(), 1);
+ assertEquals(registry.getJar(0).getName(), default_binary_name);
}
@Test
@@ -445,7 +537,7 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
}
@Test
- public void testRegistryAreaCleanUpOnFail() throws Exception {
+ public void testRegistrationFailDuringRegistryUpdate() throws Exception {
final RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
final FileSystem fs = remoteFunctionRegistry.getFs();
final String errorMessage = "Failure during remote registry update.";
[06/14] drill git commit: DRILL-5048: Fix type mismatch error in case
statement with null timestamp
Posted by su...@apache.org.
DRILL-5048: Fix type mismatch error in case statement with null timestamp
closes #657
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/85a96518
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/85a96518
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/85a96518
Branch: refs/heads/master
Commit: 85a9651844c3e5f33db14c57b8f43f01100a729e
Parents: 3cd2f44
Author: Serhii-Harnyk <se...@gmail.com>
Authored: Thu Nov 17 14:23:58 2016 +0000
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:02 2016 -0800
----------------------------------------------------------------------
.../exec/planner/logical/DrillConstExecutor.java | 3 ++-
.../exec/planner/logical/TestCaseNullableTypes.java | 15 +++++++++++++++
2 files changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/85a96518/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
index 4a0f8fa..829db2a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
@@ -287,7 +287,8 @@ public class DrillConstExecutor implements RelOptPlanner.Executor {
Calendar value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ?
new DateTime(((NullableTimeStampHolder) output).value, DateTimeZone.UTC).toCalendar(null) :
new DateTime(((TimeStampHolder) output).value, DateTimeZone.UTC).toCalendar(null);
- return rexBuilder.makeTimestampLiteral(value, 0);
+ return rexBuilder.makeLiteral(value,
+ TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.TIMESTAMP, newCall.getType().isNullable()), false);
}
case INTERVALYEAR: {
BigDecimal value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ?
http://git-wip-us.apache.org/repos/asf/drill/blob/85a96518/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestCaseNullableTypes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestCaseNullableTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestCaseNullableTypes.java
index 8e5245f..4db59eb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestCaseNullableTypes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestCaseNullableTypes.java
@@ -17,6 +17,8 @@
package org.apache.drill.exec.planner.logical;
import org.apache.drill.BaseTestQuery;
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
import org.junit.Test;
/**
@@ -124,4 +126,17 @@ public class TestCaseNullableTypes extends BaseTestQuery {
.baselineValues(1, 0.1f)
.go();
}
+
+ @Test //DRILL-5048
+ public void testCaseNullableTimestamp() throws Exception {
+ DateTime date = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")
+ .parseDateTime("2016-11-17 14:43:23");
+
+ testBuilder()
+ .sqlQuery("SELECT (CASE WHEN (false) THEN null ELSE CAST('2016-11-17 14:43:23' AS TIMESTAMP) END) res FROM (values(1)) foo")
+ .unOrdered()
+ .baselineColumns("res")
+ .baselineValues(date)
+ .go();
+ }
}
[11/14] drill git commit: DRILL-4987: Use ImpersonationUtil to get process user’s groups in RemoteFunctionRegistry
Posted by su...@apache.org.
DRILL-4987: Use ImpersonationUtil to get process user\u2019s groups in RemoteFunctionRegistry
closes #642
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a33a1858
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a33a1858
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a33a1858
Branch: refs/heads/master
Commit: a33a1858949cbac3a9c2f636a02c0b7c5bc25906
Parents: b656128
Author: Sudheesh Katkam <sk...@maprtech.com>
Authored: Tue Nov 1 13:42:52 2016 -0700
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:04 2016 -0800
----------------------------------------------------------------------
.../fn/registry/RemoteFunctionRegistry.java | 21 ++++++++++++--------
.../drill/exec/util/ImpersonationUtil.java | 9 +++++++++
2 files changed, 22 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a33a1858/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
index e5e62eb..fe79583 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.expr.fn.registry;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.DrillRuntimeException;
@@ -36,13 +36,13 @@ import org.apache.drill.exec.store.sys.PersistentStore;
import org.apache.drill.exec.store.sys.PersistentStoreConfig;
import org.apache.drill.exec.store.sys.PersistentStoreProvider;
import org.apache.drill.exec.store.sys.store.DataChangeVersion;
+import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.UserGroupInformation;
import java.io.File;
import java.io.IOException;
@@ -229,20 +229,25 @@ public class RemoteFunctionRegistry implements AutoCloseable {
Preconditions.checkState(fs.exists(path), "Area [%s] must exist", fullPath);
FileStatus fileStatus = fs.getFileStatus(path);
Preconditions.checkState(fileStatus.isDirectory(), "Area [%s] must be a directory", fullPath);
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
FsPermission permission = fileStatus.getPermission();
- // It is considered that current user has write rights on directory if:
- // 1. current user is owner of the directory and has write rights
- // 2. current user is in group that has write rights
+ // It is considered that process user has write rights on directory if:
+ // 1. process user is owner of the directory and has write rights
+ // 2. process user is in group that has write rights
// 3. any user has write rights
Preconditions.checkState(
- (currentUser.getUserName().equals(fileStatus.getOwner())
+ (ImpersonationUtil.getProcessUserName()
+ .equals(fileStatus.getOwner())
&& permission.getUserAction().implies(FsAction.WRITE)) ||
- (Lists.newArrayList(currentUser.getGroupNames()).contains(fileStatus.getGroup())
+ (Sets.newHashSet(ImpersonationUtil.getProcessUserGroupNames())
+ .contains(fileStatus.getGroup())
&& permission.getGroupAction().implies(FsAction.WRITE)) ||
permission.getOtherAction().implies(FsAction.WRITE),
"Area [%s] must be writable and executable for application user", fullPath);
} catch (Exception e) {
+ if (e instanceof DrillRuntimeException) {
+ throw (DrillRuntimeException) e;
+ }
+ // throws
DrillRuntimeException.format(e, "Error during udf area creation [%s] on file system [%s]", fullPath, fs.getUri());
}
logger.info("Created remote udf area [{}] on file system [{}]", fullPath, fs.getUri());
http://git-wip-us.apache.org/repos/asf/drill/blob/a33a1858/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
index 7790043..93ee7a0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
@@ -174,6 +174,15 @@ public class ImpersonationUtil {
}
/**
+ * Return the list of groups to which the process user belongs.
+ *
+ * @return Drillbit process user group names
+ */
+ public static String[] getProcessUserGroupNames() {
+ return getProcessUserUGI().getGroupNames();
+ }
+
+ /**
* Return the {@link org.apache.hadoop.security.UserGroupInformation} of user who is running the Drillbit.
*
* @return Drillbit process user {@link org.apache.hadoop.security.UserGroupInformation}.
[12/14] drill git commit: DRILL-5081: Lower logging level for corrupt
dates message
Posted by su...@apache.org.
DRILL-5081: Lower logging level for corrupt dates message
* introduced in DRILL-4203
closes #691
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/94f3ba0f
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/94f3ba0f
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/94f3ba0f
Branch: refs/heads/master
Commit: 94f3ba0f1398c75d1188683716155c4444836260
Parents: a33a185
Author: Sudheesh Katkam <su...@apache.org>
Authored: Tue Nov 29 13:07:36 2016 -0800
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:04 2016 -0800
----------------------------------------------------------------------
.../drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java | 4 +++-
.../main/java/org/apache/drill/exec/store/parquet/Metadata.java | 4 +++-
.../apache/drill/exec/store/parquet/ParquetScanBatchCreator.java | 4 +++-
3 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/94f3ba0f/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java
index 81be529..42db5d0 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java
@@ -127,7 +127,9 @@ public class HiveDrillNativeScanBatchCreator implements BatchCreator<HiveDrillNa
// in the first row group
ParquetReaderUtility.DateCorruptionStatus containsCorruptDates =
ParquetReaderUtility.detectCorruptDates(parquetMetadata, config.getColumns(), true);
- logger.info(containsCorruptDates.toString());
+ if (logger.isDebugEnabled()) {
+ logger.debug(containsCorruptDates.toString());
+ }
readers.add(new ParquetRecordReader(
context,
Path.getPathWithoutSchemeAndAuthority(finalPath).toString(),
http://git-wip-us.apache.org/repos/asf/drill/blob/94f3ba0f/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
index d819562..48b4497 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
@@ -404,7 +404,9 @@ public class Metadata {
ALL_COLS.add(AbstractRecordReader.STAR_COLUMN);
boolean autoCorrectCorruptDates = formatConfig.autoCorrectCorruptDates;
ParquetReaderUtility.DateCorruptionStatus containsCorruptDates = ParquetReaderUtility.detectCorruptDates(metadata, ALL_COLS, autoCorrectCorruptDates);
- logger.info(containsCorruptDates.toString());
+ if (logger.isDebugEnabled()) {
+ logger.debug(containsCorruptDates.toString());
+ }
for (BlockMetaData rowGroup : metadata.getBlocks()) {
List<ColumnMetadata_v3> columnMetadataList = Lists.newArrayList();
long length = 0;
http://git-wip-us.apache.org/repos/asf/drill/blob/94f3ba0f/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index fa7f44e..5e22458 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -116,7 +116,9 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
boolean autoCorrectCorruptDates = rowGroupScan.formatConfig.autoCorrectCorruptDates;
ParquetReaderUtility.DateCorruptionStatus containsCorruptDates = ParquetReaderUtility.detectCorruptDates(footers.get(e.getPath()), rowGroupScan.getColumns(),
autoCorrectCorruptDates);
- logger.info(containsCorruptDates.toString());
+ if (logger.isDebugEnabled()) {
+ logger.debug(containsCorruptDates.toString());
+ }
if (!context.getOptions().getOption(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val && !isComplex(footers.get(e.getPath()))) {
readers.add(
new ParquetRecordReader(