You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by dz...@apache.org on 2022/10/21 11:49:13 UTC
[drill] branch 1.20 updated (d8ab5d7453 -> 3a23665775)
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a change to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
from d8ab5d7453 [maven-release-plugin] prepare for next development iteration
new 64c4f25a19 DRILL-8275: Prevent the JDBC Client from creating spurious paths in Zookeeper (#2617)
new e5ecc5dc97 More unit tests covering the remaining column types affected by DRILL-8273. (#2620)
new 7165163d8b DRILL-8278: The period character '.' is broken in SQL LIKE patterns (#2621)
new e6f67d2541 DRILL-8280: Cannot ANALYZE files containing non-ASCII column names (#2625)
new 86f7eab40b DRILL-8281: Info schema LIKE with ESCAPE push down bug. (#2627)
new 0d0708989e DRILL-8282: Bump Hadoop-Common Version to 3.2.4 (CVE) (#2630)
new fda97be16b [MINOR UPDATE]: Disable lgtm build (#2631)
new 2a389176c6 DRILL-8283: Add a configurable recursive file listing size limit (#2632)
new 3a23665775 DRILL-8282: Update hadoop.dll and winutils.exe to 3.2.4. (#2635)
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 2 +-
.lgtm.yml | 22 --
.../store/hdf5/writers/HDF5ByteDataWriter.java | 4 +-
.../store/hdf5/writers/HDF5DoubleDataWriter.java | 4 +-
.../store/hdf5/writers/HDF5FloatDataWriter.java | 4 +-
.../exec/store/hdf5/writers/HDF5IntDataWriter.java | 4 +-
.../store/hdf5/writers/HDF5LongDataWriter.java | 4 +-
.../store/hdf5/writers/HDF5SmallIntDataWriter.java | 4 +-
.../hdf5/writers/HDF5TimestampDataWriter.java | 4 +-
.../exec/store/mapr/TableFormatPluginConfig.java | 2 +-
contrib/storage-hbase/pom.xml | 12 +
contrib/storage-hive/core/pom.xml | 26 +-
.../apache/drill/exec/udfs/CryptoFunctions.java | 4 +-
distribution/pom.xml | 4 +
.../src/main/resources/core-site-example.xml | 16 +-
.../src/main/resources/winutils/hadoop.dll | Bin 88576 -> 88576 bytes
.../src/main/resources/winutils/winutils.exe | Bin 118784 -> 118784 bytes
drill-yarn/pom.xml | 4 +
exec/java-exec/pom.xml | 48 ++++
.../drill/exec/coord/zk/ZKClusterCoordinator.java | 36 ++-
.../expr/fn/impl/CollectListMapsAggFunction.java | 2 +-
.../apache/drill/exec/expr/fn/impl/RegexpUtil.java | 2 +-
.../drill/exec/expr/fn/impl/SchemaFunctions.java | 9 +-
.../java/org/apache/drill/exec/record/DeadBuf.java | 2 +-
.../org/apache/drill/exec/server/Drillbit.java | 3 +-
.../drill/exec/store/ischema/InfoSchemaFilter.java | 26 +-
.../org/apache/drill/exec/util/FileSystemUtil.java | 71 +++++-
.../drill/exec/client/DrillClientStateTest.java | 43 ++++
.../drill/exec/expr/fn/impl/TestSqlPatterns.java | 9 +-
.../resultSet/project/TestTupleProjection.java | 34 ++-
.../org/apache/drill/exec/sql/TestInfoSchema.java | 270 ++++++++++++---------
.../drill/exec/sql/TestMetastoreCommands.java | 193 +++++++++------
.../apache/drill/exec/util/FileSystemUtilTest.java | 18 ++
exec/rpc/pom.xml | 10 +
exec/vector/pom.xml | 10 +
.../exec/record/metadata/AbstractPropertied.java | 2 +-
.../drill/exec/util/JsonStringArrayList.java | 2 +-
.../apache/drill/exec/util/JsonStringHashMap.java | 2 +-
logical/pom.xml | 8 +
metastore/iceberg-metastore/pom.xml | 12 +
metastore/metastore-api/pom.xml | 8 +
pom.xml | 91 ++++++-
42 files changed, 750 insertions(+), 281 deletions(-)
delete mode 100644 .lgtm.yml
mode change 100644 => 100755 distribution/src/main/resources/winutils/hadoop.dll
mode change 100644 => 100755 distribution/src/main/resources/winutils/winutils.exe
create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/client/DrillClientStateTest.java
[drill] 03/09: DRILL-8278: The period character '.' is broken in SQL LIKE patterns (#2621)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7165163d8b300fb6079cca49bc18ad8fe9b78b8d
Author: James Turton <91...@users.noreply.github.com>
AuthorDate: Sat Aug 13 14:23:15 2022 +0200
DRILL-8278: The period character '.' is broken in SQL LIKE patterns (#2621)
---
.../main/java/org/apache/drill/exec/expr/fn/impl/RegexpUtil.java | 2 +-
.../java/org/apache/drill/exec/expr/fn/impl/TestSqlPatterns.java | 9 ++++++++-
2 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/RegexpUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/RegexpUtil.java
index a7f63cd5a7..d86e2588e9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/RegexpUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/RegexpUtil.java
@@ -28,7 +28,7 @@ package org.apache.drill.exec.expr.fn.impl;
*/
public class RegexpUtil {
- private static final String JAVA_REGEX_SPECIALS = "[]()|^-+*?{}$\\";
+ private static final String JAVA_REGEX_SPECIALS = "[]()|^-+*?{}$\\.";
private static final String SQL_SIMILAR_SPECIALS = "[]()|^-+*_%?{}";
private static final String [] REG_CHAR_CLASSES = {
"[:ALPHA:]", "\\p{Alpha}",
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestSqlPatterns.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestSqlPatterns.java
index 9e5aae6c16..73db8ca739 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestSqlPatterns.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestSqlPatterns.java
@@ -159,6 +159,13 @@ public class TestSqlPatterns extends BaseTest {
assertEquals("AB", patternInfo.getSimplePatternString());
assertEquals(RegexpUtil.SqlPatternType.CONSTANT, patternInfo.getPatternType());
+ // A.B is constant. DRILL-8278
+ patternInfo = RegexpUtil.sqlToRegexLike("A.B");
+ // The . should be escaped with a \ so that it represents a literal .
+ assertEquals("A\\.B", patternInfo.getJavaPatternString());
+ assertEquals("A.B", patternInfo.getSimplePatternString());
+ assertEquals(RegexpUtil.SqlPatternType.CONSTANT, patternInfo.getPatternType());
+
// Test with escape characters.
// A%#B is invalid escape sequence
@@ -449,7 +456,7 @@ public class TestSqlPatterns extends BaseTest {
}
@Test
- public void testSqlPatternContainsMUltipleMatchers() {
+ public void testSqlPatternContainsMultipleMatchers() {
final String longASCIIString = "Drill supports a variety of NoSQL databases and file systems, including HBase, MongoDB, MapR-DB, HDFS, MapR-FS, Amazon S3, Azure Blob Storage, Google Cloud Storage, Swift, "
+ "NAS and local files. A single query can join data from multiple datastores. For example, you can join a user profile collection in MongoDB with a directory of event logs in Hadoop.";
[drill] 05/09: DRILL-8281: Info schema LIKE with ESCAPE push down bug. (#2627)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 86f7eab40bb75c0ab485f83261210cd6562dac82
Author: James Turton <91...@users.noreply.github.com>
AuthorDate: Mon Aug 22 05:53:09 2022 +0200
DRILL-8281: Info schema LIKE with ESCAPE push down bug. (#2627)
---
.../drill/exec/store/ischema/InfoSchemaFilter.java | 26 +-
.../org/apache/drill/exec/sql/TestInfoSchema.java | 270 ++++++++++++---------
2 files changed, 174 insertions(+), 122 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilter.java
index bc71c98c0b..ebf6c19e47 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilter.java
@@ -147,7 +147,7 @@ public class InfoSchemaFilter {
* Evaluate the filter for given <COLUMN NAME, VALUE> pairs.
*
* @param recordValues map of field names and their values
- * @param prefixMatchesInconclusive whether a prefix match between a column value and a filter value
+ * @param prefixMatchesInconclusive whether a prefix match between a schema path and a filter value
* results in Result.INCONCLUSIVE. Used for pruning the schema search
* tree, e.g. "dfs" need not be recursed to find a schema of "cp.default"
* @return evaluation result
@@ -194,15 +194,23 @@ public class InfoSchemaFilter {
: sqlToRegexLike(pattern.value, escape.value);
if (Pattern.matches(spi.getJavaPatternString(), fieldValue)) {
+ // E.g. pattern = 'dfs.%', schema path = 'dfs.tmp'.
+ // The entire schema path matches.
return Result.TRUE;
}
if (!prefixMatchesInconclusive) {
+ // E.g. pattern = 'dfs.%', schema path = 'dfs'.
+ // There may be prefix match but prefixMatchesInconclusive is false.
return Result.FALSE;
}
if ((spi.getPatternType() == SqlPatternType.STARTS_WITH || spi.getPatternType() == SqlPatternType.CONSTANT) &&
- !pattern.value.startsWith(fieldValue)) {
+ !spi.getSimplePatternString().startsWith(fieldValue)) {
+ // E.g. pattern = 'dfs.%', schema path = 'cp'.
+ // No match, not even to a prefix.
return Result.FALSE;
}
+ // E.g. pattern = 'dfs.%', schema path = 'dfs'.
+ // A prefix matches
return Result.INCONCLUSIVE;
}
case FunctionNames.EQ:
@@ -220,24 +228,18 @@ public class InfoSchemaFilter {
boolean exactMatch = prefixMatch && arg.value.equals(value);
if (exprNode.function.equals(FunctionNames.EQ)) {
+ // Equality case
if (exactMatch) {
return Result.TRUE;
} else {
- if (prefixMatchesInconclusive && prefixMatch) {
- return Result.INCONCLUSIVE;
- } else {
- return Result.FALSE;
- }
+ return prefixMatchesInconclusive && prefixMatch ? Result.INCONCLUSIVE: Result.FALSE;
}
} else {
+ // Inequality case
if (exactMatch) {
return Result.FALSE;
} else {
- if (prefixMatchesInconclusive && prefixMatch) {
- return Result.INCONCLUSIVE;
- } else {
- return Result.TRUE;
- }
+ return prefixMatchesInconclusive && prefixMatch ? Result.INCONCLUSIVE : Result.TRUE;
}
}
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java
index fa616e6623..469462e601 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java
@@ -18,16 +18,23 @@
package org.apache.drill.exec.sql;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.drill.PlanTestBase;
import org.apache.drill.categories.SqlTest;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.apache.drill.common.exceptions.UserRemoteException;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.physical.rowSet.DirectRowSet;
+import org.apache.drill.exec.physical.rowSet.RowSet;
+import org.apache.drill.exec.physical.rowSet.RowSetBuilder;
+import org.apache.drill.exec.physical.rowSet.RowSetReader;
+import org.apache.drill.exec.record.metadata.SchemaBuilder;
+import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.dfs.FileSystemConfig;
-import org.apache.drill.exec.vector.NullableVarCharVector;
-import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.ClusterFixtureBuilder;
+import org.apache.drill.test.ClusterTest;
+import org.apache.drill.test.QueryBuilder;
import org.apache.drill.test.TestBuilder;
+import org.apache.drill.test.rowSet.RowSetComparison;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -41,8 +48,10 @@ import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT;
import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.CATS_COL_CATALOG_CONNECT;
import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.CATS_COL_CATALOG_DESCRIPTION;
import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.CATS_COL_CATALOG_NAME;
+import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
/**
@@ -52,25 +61,27 @@ import static org.junit.Assert.assertTrue;
* -- USE schema
*/
@Category(SqlTest.class)
-public class TestInfoSchema extends BaseTestQuery {
+public class TestInfoSchema extends ClusterTest {
private static final String TEST_SUB_DIR = "testSubDir";
private static final ObjectMapper mapper = new ObjectMapper().enable(INDENT_OUTPUT);
@BeforeClass
- public static void setupFiles() {
+ public static void setupFiles() throws Exception {
+ ClusterTest.startCluster(new ClusterFixtureBuilder(dirTestWatcher));
dirTestWatcher.copyFileToRoot(Paths.get("sample-data"));
dirTestWatcher.makeRootSubDir(Paths.get(TEST_SUB_DIR));
+
}
@Test
- public void selectFromAllTables() throws Exception{
- test("select * from INFORMATION_SCHEMA.SCHEMATA");
- test("select * from INFORMATION_SCHEMA.CATALOGS");
- test("select * from INFORMATION_SCHEMA.VIEWS");
- test("select * from INFORMATION_SCHEMA.`TABLES`");
- test("select * from INFORMATION_SCHEMA.COLUMNS");
- test("select * from INFORMATION_SCHEMA.`FILES`");
- test("select * from INFORMATION_SCHEMA.`PARTITIONS`");
+ public void selectFromAllTables() throws Exception {
+ runAndLog("select * from INFORMATION_SCHEMA.SCHEMATA");
+ runAndLog("select * from INFORMATION_SCHEMA.CATALOGS");
+ runAndLog("select * from INFORMATION_SCHEMA.VIEWS");
+ runAndLog("select * from INFORMATION_SCHEMA.`TABLES`");
+ runAndLog("select * from INFORMATION_SCHEMA.COLUMNS");
+ runAndLog("select * from INFORMATION_SCHEMA.`FILES`");
+ runAndLog("select * from INFORMATION_SCHEMA.`PARTITIONS`");
}
@Test
@@ -84,7 +95,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void showTablesFromDb() throws Exception{
+ public void showTablesFromDb() throws Exception {
List<String[]> expected = Arrays.asList(
new String[]{"information_schema", "VIEWS"},
new String[]{"information_schema", "COLUMNS"},
@@ -114,7 +125,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void showTablesFromDbWhere() throws Exception{
+ public void showTablesFromDbWhere() throws Exception {
testBuilder()
.sqlQuery("SHOW TABLES FROM INFORMATION_SCHEMA WHERE TABLE_NAME='VIEWS'")
.unOrdered()
@@ -124,7 +135,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void showTablesLike() throws Exception{
+ public void showTablesLike() throws Exception {
testBuilder()
.sqlQuery("SHOW TABLES LIKE '%CH%'")
.unOrdered()
@@ -135,26 +146,37 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void showDatabases() throws Exception{
- List<String> expected = Arrays.asList("dfs.default", "dfs.root", "dfs.tmp", "cp.default", "sys", "information_schema");
-
- TestBuilder t1 = testBuilder()
- .sqlQuery("SHOW DATABASES")
- .unOrdered()
- .baselineColumns("SCHEMA_NAME");
- expected.forEach(t1::baselineValues);
- t1.go();
+ public void showDatabases() throws Exception {
+ QueryBuilder builder = client.queryBuilder().sql("SHOW DATABASES");
+ DirectRowSet sets = builder.rowSet();
- TestBuilder t2 = testBuilder()
- .sqlQuery("SHOW SCHEMAS")
- .unOrdered()
- .baselineColumns("SCHEMA_NAME");
- expected.forEach(t2::baselineValues);
- t2.go();
+ try {
+ TupleMetadata schema = new SchemaBuilder()
+ .addNullable("SCHEMA_NAME", TypeProtos.MinorType.VARCHAR)
+ .build();
+
+ RowSet expected = new RowSetBuilder(client.allocator(), schema)
+ .addRow("dfs.default")
+ .addRow("dfs.root")
+ .addRow("dfs.tmp")
+ .addRow("cp.default")
+ .addRow("sys")
+ .addRow("information_schema")
+ .addRow("mock")
+ .build();
+
+ new RowSetComparison(expected).unorderedVerifyAndClearAll(sets);
+
+ builder = client.queryBuilder().sql("SHOW SCHEMAS");
+ sets = builder.rowSet();
+ new RowSetComparison(expected).unorderedVerifyAndClearAll(sets);
+ } finally {
+ sets.clear();
+ }
}
@Test
- public void showDatabasesWhere() throws Exception{
+ public void showDatabasesWhere() throws Exception {
testBuilder()
.sqlQuery("SHOW DATABASES WHERE SCHEMA_NAME='dfs.tmp'")
.unOrdered()
@@ -175,17 +197,41 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void showDatabasesLike() throws Exception{
+ public void showDatabasesLike() throws Exception {
testBuilder()
.sqlQuery("SHOW DATABASES LIKE '%y%'")
.unOrdered()
.baselineColumns("SCHEMA_NAME")
.baselineValues("sys")
.go();
+
+
+ }
+
+ @Test // DRILL-8281
+ public void likePatternWithEscapeChar() throws Exception {
+ StoragePluginRegistry pluginRegistry = cluster.drillbit().getContext().getStorage();
+ pluginRegistry.validatedPut(
+ "dfs_with_underscore",
+ pluginRegistry.getDefinedConfig("dfs")
+ );
+
+ try {
+ testBuilder()
+ .sqlQuery("SHOW DATABASES WHERE schema_name LIKE 'dfs^_with^_underscore.%' escape '^'")
+ .unOrdered()
+ .baselineColumns("SCHEMA_NAME")
+ .baselineValues("dfs_with_underscore.default")
+ .baselineValues("dfs_with_underscore.tmp")
+ .baselineValues("dfs_with_underscore.root")
+ .go();
+ } finally {
+ pluginRegistry.remove("dfs_with_underscore");
+ }
}
@Test
- public void describeTable() throws Exception{
+ public void describeTable() throws Exception {
testBuilder()
.sqlQuery("DESCRIBE CATALOGS")
.unOrdered()
@@ -199,7 +245,7 @@ public class TestInfoSchema extends BaseTestQuery {
@Test
public void describeTableWithTableKeyword() throws Exception {
- test("USE INFORMATION_SCHEMA");
+ runAndLog("USE INFORMATION_SCHEMA");
testBuilder()
.sqlQuery("DESCRIBE TABLE CATALOGS")
.unOrdered()
@@ -208,7 +254,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void describeTableWithSchema() throws Exception{
+ public void describeTableWithSchema() throws Exception {
testBuilder()
.sqlQuery("DESCRIBE INFORMATION_SCHEMA.`TABLES`")
.unOrdered()
@@ -234,10 +280,10 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void describeWhenSameTableNameExistsInMultipleSchemas() throws Exception{
+ public void describeWhenSameTableNameExistsInMultipleSchemas() throws Exception {
try {
- test("USE dfs.tmp");
- test("CREATE OR REPLACE VIEW `TABLES` AS SELECT full_name FROM cp.`employee.json`");
+ runAndLog("USE dfs.tmp");
+ runAndLog("CREATE OR REPLACE VIEW `TABLES` AS SELECT full_name FROM cp.`employee.json`");
testBuilder()
.sqlQuery("DESCRIBE `TABLES`")
@@ -261,15 +307,15 @@ public class TestInfoSchema extends BaseTestQuery {
.baselineValues("LAST_MODIFIED_TIME", "TIMESTAMP", "NO")
.go();
} finally {
- test("DROP VIEW IF EXISTS dfs.tmp.`TABLES`");
+ runAndLog("DROP VIEW IF EXISTS dfs.tmp.`TABLES`");
}
}
@Test
public void describeWhenSameTableNameExistsInMultipleSchemasWithTableKeyword() throws Exception {
try {
- test("USE dfs.tmp");
- test("CREATE OR REPLACE VIEW `TABLES` AS SELECT full_name FROM cp.`employee.json`");
+ runAndLog("USE dfs.tmp");
+ runAndLog("CREATE OR REPLACE VIEW `TABLES` AS SELECT full_name FROM cp.`employee.json`");
testBuilder()
.sqlQuery("DESCRIBE TABLE `TABLES`")
@@ -283,12 +329,12 @@ public class TestInfoSchema extends BaseTestQuery {
.sqlBaselineQuery("DESCRIBE INFORMATION_SCHEMA.`TABLES`")
.go();
} finally {
- test("DROP VIEW IF EXISTS dfs.tmp.`TABLES`");
+ runAndLog("DROP VIEW IF EXISTS dfs.tmp.`TABLES`");
}
}
@Test
- public void describeTableWithColumnName() throws Exception{
+ public void describeTableWithColumnName() throws Exception {
testBuilder()
.sqlQuery("DESCRIBE `TABLES` TABLE_CATALOG")
.unOrdered()
@@ -300,7 +346,7 @@ public class TestInfoSchema extends BaseTestQuery {
@Test
public void describeTableWithColumnNameAndTableKeyword() throws Exception {
- test("USE INFORMATION_SCHEMA");
+ runAndLog("USE INFORMATION_SCHEMA");
testBuilder()
.sqlQuery("DESCRIBE TABLE `TABLES` TABLE_CATALOG")
.unOrdered()
@@ -309,7 +355,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void describeTableWithSchemaAndColumnName() throws Exception{
+ public void describeTableWithSchemaAndColumnName() throws Exception {
testBuilder()
.sqlQuery("DESCRIBE INFORMATION_SCHEMA.`TABLES` TABLE_CATALOG")
.unOrdered()
@@ -328,7 +374,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void describeTableWithColQualifier() throws Exception{
+ public void describeTableWithColQualifier() throws Exception {
testBuilder()
.sqlQuery("DESCRIBE COLUMNS 'TABLE%'")
.unOrdered()
@@ -341,7 +387,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void describeTableWithSchemaAndColQualifier() throws Exception{
+ public void describeTableWithSchemaAndColQualifier() throws Exception {
testBuilder()
.sqlQuery("DESCRIBE INFORMATION_SCHEMA.SCHEMATA 'SCHEMA%'")
.unOrdered()
@@ -352,7 +398,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void defaultSchemaDfs() throws Exception{
+ public void defaultSchemaDfs() throws Exception {
testBuilder()
.sqlQuery("SELECT R_REGIONKEY FROM `sample-data/region.parquet` LIMIT 1")
.unOrdered()
@@ -363,7 +409,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void defaultSchemaClasspath() throws Exception{
+ public void defaultSchemaClasspath() throws Exception {
testBuilder()
.sqlQuery("SELECT full_name FROM `employee.json` LIMIT 1")
.unOrdered()
@@ -375,7 +421,7 @@ public class TestInfoSchema extends BaseTestQuery {
@Test
- public void queryFromNonDefaultSchema() throws Exception{
+ public void queryFromNonDefaultSchema() throws Exception {
testBuilder()
.sqlQuery("SELECT full_name FROM cp.`employee.json` LIMIT 1")
.unOrdered()
@@ -386,7 +432,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void useSchema() throws Exception{
+ public void useSchema() throws Exception {
testBuilder()
.sqlQuery("USE dfs.`default`")
.unOrdered()
@@ -396,7 +442,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void useSubSchemaWithinSchema() throws Exception{
+ public void useSubSchemaWithinSchema() throws Exception {
testBuilder()
.sqlQuery("USE dfs")
.unOrdered()
@@ -420,89 +466,93 @@ public class TestInfoSchema extends BaseTestQuery {
}
@Test
- public void useSchemaNegative() throws Exception{
- errorMsgTestHelper("USE invalid.schema",
- "Schema [invalid.schema] is not valid with respect to either root schema or current default schema.");
+ public void useSchemaNegative() throws Exception {
+ try {
+ queryBuilder().sql("USE invalid.schema").run();
+ } catch (UserRemoteException e) {
+ assertThat(
+ e.getMessage(),
+ containsString("Schema [invalid.schema] is not valid with respect to either root schema or current default schema.")
+ );
+ }
}
// Tests using backticks around the complete schema path
// select * from `dfs.tmp`.`/tmp/nation.parquet`;
@Test
public void completeSchemaRef1() throws Exception {
- test("SELECT * FROM `cp.default`.`employee.json` limit 2");
+ runAndLog("SELECT * FROM `cp.default`.`employee.json` limit 2");
}
@Test
public void describeSchemaSyntax() throws Exception {
- test("describe schema dfs");
- test("describe schema dfs.`default`");
- test("describe database dfs.`default`");
+ runAndLog("describe schema dfs");
+ runAndLog("describe schema dfs.`default`");
+ runAndLog("describe database dfs.`default`");
}
@Test
public void describePartialSchema() throws Exception {
- test("use dfs");
- test("describe schema tmp");
+ runAndLog("use dfs");
+ runAndLog("describe schema tmp");
}
@Test
public void describeSchemaOutput() throws Exception {
- List<QueryDataBatch> result = testSqlWithResults("describe schema dfs.tmp");
- assertEquals(1, result.size());
- QueryDataBatch batch = result.get(0);
- RecordBatchLoader loader = new RecordBatchLoader(getDrillbitContext().getAllocator());
- loader.load(batch.getHeader().getDef(), batch.getData());
-
- // check schema column value
- VectorWrapper<?> schemaValueVector = loader.getValueAccessorById(
- NullableVarCharVector.class,
- loader.getValueVectorId(SchemaPath.getCompoundPath("schema")).getFieldIds());
- String schema = schemaValueVector.getValueVector().getAccessor().getObject(0).toString();
- assertEquals("dfs.tmp", schema);
-
- // check properties column value
- VectorWrapper<?> propertiesValueVector = loader.getValueAccessorById(
- NullableVarCharVector.class,
- loader.getValueVectorId(SchemaPath.getCompoundPath("properties")).getFieldIds());
- String properties = propertiesValueVector.getValueVector().getAccessor().getObject(0).toString();
- Map<?, ?> configMap = mapper.readValue(properties, Map.class);
-
- // check some stable properties existence
- assertTrue(configMap.containsKey("connection"));
- assertTrue(configMap.containsKey("formats"));
- assertFalse(configMap.containsKey("workspaces"));
-
- // check some stable properties values
- assertEquals("file", configMap.get("type"));
-
- FileSystemConfig testConfig = (FileSystemConfig) bits[0].getContext().getStorage().getPlugin("dfs").getConfig();
- String tmpSchemaLocation = testConfig.getWorkspaces().get("tmp").getLocation();
- assertEquals(tmpSchemaLocation, configMap.get("location"));
-
- batch.release();
- loader.clear();
+ QueryBuilder builder = queryBuilder().sql("describe schema dfs.tmp");
+ DirectRowSet rows = builder.rowSet();
+ try {
+ assertEquals(1, rows.rowCount());
+
+ RowSetReader reader = rows.reader();
+ assertTrue(reader.next());
+ String schema = (String) reader.column(0).reader().getObject();
+ assertEquals("dfs.tmp", schema);
+
+ String properties = (String) reader.column(1).reader().getObject();
+ Map<?, ?> configMap = mapper.readValue(properties, Map.class);
+ // check some stable properties existence
+ assertTrue(configMap.containsKey("connection"));
+ assertTrue(configMap.containsKey("formats"));
+ assertFalse(configMap.containsKey("workspaces"));
+
+ // check some stable properties values
+ assertEquals("file", configMap.get("type"));
+
+ DrillbitContext ctx = cluster.drillbit().getContext();
+ FileSystemConfig testConfig = (FileSystemConfig) ctx.getStorage().getPlugin("dfs").getConfig();
+ String tmpSchemaLocation = testConfig.getWorkspaces().get("tmp").getLocation();
+ assertEquals(tmpSchemaLocation, configMap.get("location"));
+ } finally {
+ rows.clear();
+ }
}
@Test
public void describeSchemaInvalid() throws Exception {
- errorMsgTestHelper("describe schema invalid.schema", "Invalid schema name [invalid.schema]");
+ try {
+ queryBuilder().sql("describe schema invalid.schema").run();
+ } catch (UserRemoteException e) {
+ assertThat(
+ e.getMessage(),
+ containsString("Invalid schema name [invalid.schema]")
+ );
+ }
}
@Test
public void testDescribeAlias() throws Exception {
- test("desc schema dfs.tmp");
- test("desc information_schema.`catalogs`");
- test("desc table information_schema.`catalogs`");
+ runAndLog("desc schema dfs.tmp");
+ runAndLog("desc information_schema.`catalogs`");
+ runAndLog("desc table information_schema.`catalogs`");
}
@Test
public void testSerDe() throws Exception {
- String sql = "select * from information_schema.`tables` where table_name = 'schemata' order by 1";
+ String sql = "select count() from information_schema.`tables` where table_name = 'SCHEMATA'";
+ String plan = queryBuilder().sql(sql).explainJson();
+ long cnt = queryBuilder().physical(plan).singletonLong();
- testBuilder()
- .sqlQuery(sql)
- .unOrdered()
- .physicalPlanBaseline(PlanTestBase.getPhysicalJsonPlan(sql))
- .go();
+ assertEquals("Counts should match", 1, cnt);
}
}
[drill] 02/09: More unit tests covering the remaining column types affected by DRILL-8273. (#2620)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit e5ecc5dc97b5c5b5ff321a743e37b5c43083aa6e
Author: James Turton <91...@users.noreply.github.com>
AuthorDate: Fri Aug 12 03:32:36 2022 +0200
More unit tests covering the remaining column types affected by DRILL-8273. (#2620)
---
.../resultSet/project/TestTupleProjection.java | 34 ++++++++++++++++++----
1 file changed, 28 insertions(+), 6 deletions(-)
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/project/TestTupleProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/project/TestTupleProjection.java
index 6aec100dc8..3efddbf170 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/project/TestTupleProjection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/project/TestTupleProjection.java
@@ -29,6 +29,7 @@ import java.util.List;
import org.apache.drill.categories.RowSetTests;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.physical.resultSet.project.RequestedTuple.TupleProjectionType;
@@ -60,13 +61,25 @@ public class TestTupleProjection extends BaseTest {
MetadataUtils.newScalar("a", Types.required(MinorType.INT));
private static final ColumnMetadata UNPROJECTED_SPECIAL_COLUMN =
MetadataUtils.newScalar("bar", Types.required(MinorType.INT));
- private static final ColumnMetadata COMPLEX_SPECIAL_COLUMN =
+ private static final ColumnMetadata SPECIAL_DICT =
+ MetadataUtils.newDict("a_dict");
+ private static final ColumnMetadata SPECIAL_MAP =
MetadataUtils.newMap("a_map");
+ private static final ColumnMetadata SPECIAL_REP_LIST =
+ MetadataUtils.newRepeatedList(
+ "a_repeated_list",
+ MetadataUtils.newScalar("child", Types.repeated(MinorType.INT))
+ );
+ private static final ColumnMetadata SPECIAL_VARIANT =
+ MetadataUtils.newVariant("a_variant", DataMode.OPTIONAL);
static {
SPECIAL_COLUMN.setBooleanProperty(ColumnMetadata.EXCLUDE_FROM_WILDCARD, true);
UNPROJECTED_SPECIAL_COLUMN.setBooleanProperty(ColumnMetadata.EXCLUDE_FROM_WILDCARD, true);
- COMPLEX_SPECIAL_COLUMN.setBooleanProperty(ColumnMetadata.EXCLUDE_FROM_WILDCARD, true);
+ SPECIAL_DICT.setBooleanProperty(ColumnMetadata.EXCLUDE_FROM_WILDCARD, true);
+ SPECIAL_MAP.setBooleanProperty(ColumnMetadata.EXCLUDE_FROM_WILDCARD, true);
+ SPECIAL_REP_LIST.setBooleanProperty(ColumnMetadata.EXCLUDE_FROM_WILDCARD, true);
+ SPECIAL_VARIANT.setBooleanProperty(ColumnMetadata.EXCLUDE_FROM_WILDCARD, true);
}
/**
@@ -79,7 +92,10 @@ public class TestTupleProjection extends BaseTest {
assertTrue(projSet.isProjected("foo"));
assertTrue(projSet.isProjected(NORMAL_COLUMN));
assertFalse(projSet.isProjected(SPECIAL_COLUMN));
- assertFalse(projSet.isProjected(COMPLEX_SPECIAL_COLUMN));
+ assertFalse(projSet.isProjected(SPECIAL_DICT));
+ assertFalse(projSet.isProjected(SPECIAL_MAP));
+ assertFalse(projSet.isProjected(SPECIAL_REP_LIST));
+ assertFalse(projSet.isProjected(SPECIAL_VARIANT));
assertTrue(projSet.projections().isEmpty());
assertFalse(projSet.isEmpty());
}
@@ -95,7 +111,10 @@ public class TestTupleProjection extends BaseTest {
assertNull(projSet.get("foo"));
assertTrue(projSet.isProjected(NORMAL_COLUMN));
assertFalse(projSet.isProjected(SPECIAL_COLUMN));
- assertFalse(projSet.isProjected(COMPLEX_SPECIAL_COLUMN));
+ assertFalse(projSet.isProjected(SPECIAL_DICT));
+ assertFalse(projSet.isProjected(SPECIAL_MAP));
+ assertFalse(projSet.isProjected(SPECIAL_REP_LIST));
+ assertFalse(projSet.isProjected(SPECIAL_VARIANT));
assertEquals(1, projSet.projections().size());
assertFalse(projSet.isEmpty());
}
@@ -112,7 +131,10 @@ public class TestTupleProjection extends BaseTest {
assertFalse(projSet.isProjected("foo"));
assertFalse(projSet.isProjected(NORMAL_COLUMN));
assertFalse(projSet.isProjected(SPECIAL_COLUMN));
- assertFalse(projSet.isProjected(COMPLEX_SPECIAL_COLUMN));
+ assertFalse(projSet.isProjected(SPECIAL_DICT));
+ assertFalse(projSet.isProjected(SPECIAL_MAP));
+ assertFalse(projSet.isProjected(SPECIAL_REP_LIST));
+ assertFalse(projSet.isProjected(SPECIAL_VARIANT));
assertTrue(projSet.projections().isEmpty());
assertTrue(projSet.isEmpty());
}
@@ -134,7 +156,7 @@ public class TestTupleProjection extends BaseTest {
assertTrue(projSet.isProjected(SPECIAL_COLUMN));
assertFalse(projSet.isProjected(UNPROJECTED_COLUMN));
assertFalse(projSet.isProjected(UNPROJECTED_SPECIAL_COLUMN));
- assertFalse(projSet.isProjected(COMPLEX_SPECIAL_COLUMN));
+ assertFalse(projSet.isProjected(SPECIAL_MAP));
List<RequestedColumn> cols = projSet.projections();
assertEquals(3, cols.size());
[drill] 07/09: [MINOR UPDATE]: Disable lgtm build (#2631)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit fda97be16beec83b189cade85f5af155de3f365d
Author: PJ Fanning <pj...@users.noreply.github.com>
AuthorDate: Thu Sep 8 22:09:48 2022 +0800
[MINOR UPDATE]: Disable lgtm build (#2631)
---
.lgtm.yml | 22 ----------------------
.../store/hdf5/writers/HDF5ByteDataWriter.java | 4 ++--
.../store/hdf5/writers/HDF5DoubleDataWriter.java | 4 ++--
.../store/hdf5/writers/HDF5FloatDataWriter.java | 4 ++--
.../exec/store/hdf5/writers/HDF5IntDataWriter.java | 4 ++--
.../store/hdf5/writers/HDF5LongDataWriter.java | 4 ++--
.../store/hdf5/writers/HDF5SmallIntDataWriter.java | 4 ++--
.../hdf5/writers/HDF5TimestampDataWriter.java | 4 ++--
.../exec/store/mapr/TableFormatPluginConfig.java | 2 +-
.../apache/drill/exec/udfs/CryptoFunctions.java | 4 ++--
.../expr/fn/impl/CollectListMapsAggFunction.java | 2 +-
.../drill/exec/expr/fn/impl/SchemaFunctions.java | 4 ++--
.../java/org/apache/drill/exec/record/DeadBuf.java | 2 +-
.../exec/record/metadata/AbstractPropertied.java | 2 +-
.../drill/exec/util/JsonStringArrayList.java | 2 +-
.../apache/drill/exec/util/JsonStringHashMap.java | 2 +-
pom.xml | 1 -
17 files changed, 24 insertions(+), 47 deletions(-)
diff --git a/.lgtm.yml b/.lgtm.yml
deleted file mode 100644
index a12bee6b43..0000000000
--- a/.lgtm.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Format of this file: https://lgtm.com/help/lgtm/lgtm.yml-configuration-file
-path_classifiers:
- documentation:
- - docs/dev/*
- excludes:
- - exec/java-exec/src/main/resources/rest/static/js/*.js
diff --git a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5ByteDataWriter.java b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5ByteDataWriter.java
index d66d6ddf31..94ce2da3cf 100644
--- a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5ByteDataWriter.java
+++ b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5ByteDataWriter.java
@@ -65,10 +65,10 @@ public class HDF5ByteDataWriter extends HDF5DataWriter {
@Override
public boolean write() {
- if (counter > data.length) {
+ if (!hasNext()) {
return false;
} else {
- colWriter.setInt(data[counter++]); //lgtm [java/index-out-of-bounds]
+ colWriter.setInt(data[counter++]);
return true;
}
}
diff --git a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5DoubleDataWriter.java b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5DoubleDataWriter.java
index 93b55c4935..e619dfc214 100644
--- a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5DoubleDataWriter.java
+++ b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5DoubleDataWriter.java
@@ -63,10 +63,10 @@ public class HDF5DoubleDataWriter extends HDF5DataWriter {
@Override
public boolean write() {
- if (counter > data.length) {
+ if (!hasNext()) {
return false;
} else {
- colWriter.setDouble(data[counter++]); // lgtm [java/index-out-of-bounds]
+ colWriter.setDouble(data[counter++]);
return true;
}
}
diff --git a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5FloatDataWriter.java b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5FloatDataWriter.java
index 418db01186..769c57fbd3 100644
--- a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5FloatDataWriter.java
+++ b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5FloatDataWriter.java
@@ -63,10 +63,10 @@ public class HDF5FloatDataWriter extends HDF5DataWriter {
@Override
public boolean write() {
- if (counter > data.length) {
+ if (!hasNext()) {
return false;
} else {
- colWriter.setDouble(data[counter++]); // lgtm [java/index-out-of-bounds]
+ colWriter.setDouble(data[counter++]);
return true;
}
}
diff --git a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5IntDataWriter.java b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5IntDataWriter.java
index ef0de85022..6eee89ecaf 100644
--- a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5IntDataWriter.java
+++ b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5IntDataWriter.java
@@ -64,10 +64,10 @@ public class HDF5IntDataWriter extends HDF5DataWriter {
@Override
public boolean write() {
- if (counter > data.length) {
+ if (!hasNext()) {
return false;
} else {
- colWriter.setInt(data[counter++]); // lgtm [java/index-out-of-bounds]
+ colWriter.setInt(data[counter++]);
return true;
}
}
diff --git a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5LongDataWriter.java b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5LongDataWriter.java
index 91d218d5f4..1fe51b04c6 100644
--- a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5LongDataWriter.java
+++ b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5LongDataWriter.java
@@ -63,10 +63,10 @@ public class HDF5LongDataWriter extends HDF5DataWriter {
@Override
public boolean write() {
- if (counter > data.length) {
+ if (!hasNext()) {
return false;
} else {
- colWriter.setLong(data[counter++]); // lgtm [java/index-out-of-bounds]
+ colWriter.setLong(data[counter++]);
return true;
}
}
diff --git a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5SmallIntDataWriter.java b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5SmallIntDataWriter.java
index 2f999bd991..d355457605 100644
--- a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5SmallIntDataWriter.java
+++ b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5SmallIntDataWriter.java
@@ -65,10 +65,10 @@ public class HDF5SmallIntDataWriter extends HDF5DataWriter {
@Override
public boolean write() {
- if (counter > data.length) {
+ if (!hasNext()) {
return false;
} else {
- colWriter.setInt(data[counter++]); // lgtm [java/index-out-of-bounds]
+ colWriter.setInt(data[counter++]);
return true;
}
}
diff --git a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5TimestampDataWriter.java b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5TimestampDataWriter.java
index bdec31a669..bb2129e6ed 100644
--- a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5TimestampDataWriter.java
+++ b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5TimestampDataWriter.java
@@ -42,10 +42,10 @@ public class HDF5TimestampDataWriter extends HDF5DataWriter {
@Override
public boolean write() {
- if (counter > data.length) {
+ if (!hasNext()) {
return false;
} else {
- colWriter.setTimestamp(Instant.ofEpochMilli(data[counter++])); // lgtm [java/index-out-of-bounds]
+ colWriter.setTimestamp(Instant.ofEpochMilli(data[counter++]));
return true;
}
}
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java
index 0122254dd3..ffd3d92150 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.store.mapr;
import org.apache.drill.common.logical.FormatPluginConfig;
public abstract class TableFormatPluginConfig implements FormatPluginConfig {
- //lgtm [java/inconsistent-equals-and-hashcode]
+
@Override
public boolean equals(Object obj) {
if (this == obj) {
diff --git a/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/CryptoFunctions.java b/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/CryptoFunctions.java
index e7040891fe..01fae080cd 100644
--- a/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/CryptoFunctions.java
+++ b/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/CryptoFunctions.java
@@ -285,7 +285,7 @@ public class CryptoFunctions {
keyByteArray = java.util.Arrays.copyOf(keyByteArray, 16);
javax.crypto.spec.SecretKeySpec secretKey = new javax.crypto.spec.SecretKeySpec(keyByteArray, "AES");
- javax.crypto.Cipher cipher = javax.crypto.Cipher.getInstance("AES/ECB/PKCS5Padding"); // lgtm [java/weak-cryptographic-algorithm]
+ javax.crypto.Cipher cipher = javax.crypto.Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(javax.crypto.Cipher.ENCRYPT_MODE, secretKey);
encryptedText = javax.xml.bind.DatatypeConverter.printBase64Binary(cipher.doFinal(input.getBytes(java.nio.charset.StandardCharsets.UTF_8)));
} catch (Exception e) {
@@ -336,7 +336,7 @@ public class CryptoFunctions {
keyByteArray = java.util.Arrays.copyOf(keyByteArray, 16);
javax.crypto.spec.SecretKeySpec secretKey = new javax.crypto.spec.SecretKeySpec(keyByteArray, "AES");
- javax.crypto.Cipher cipher = javax.crypto.Cipher.getInstance("AES/ECB/PKCS5Padding"); // lgtm [java/weak-cryptographic-algorithm]
+ javax.crypto.Cipher cipher = javax.crypto.Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(javax.crypto.Cipher.DECRYPT_MODE, secretKey);
decryptedText = new String(cipher.doFinal(javax.xml.bind.DatatypeConverter.parseBase64Binary(input)));
} catch (Exception e) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CollectListMapsAggFunction.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CollectListMapsAggFunction.java
index f88f0e4fe8..c232edc441 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CollectListMapsAggFunction.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CollectListMapsAggFunction.java
@@ -60,7 +60,7 @@ public class CollectListMapsAggFunction implements DrillAggFunc {
for (int i = 0; i < inputs.length; i += 2) {
org.apache.drill.exec.vector.complex.MapUtility.writeToMapFromReader(
- inputs[i + 1], mapWriter, inputs[i].readText().toString(), "CollectListMapsAggFunction"); // lgtm [java/index-out-of-bounds]
+ inputs[i + 1], mapWriter, inputs[i].readText().toString(), "CollectListMapsAggFunction");
}
mapWriter.end();
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SchemaFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SchemaFunctions.java
index a71447eb4a..b19905bf98 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SchemaFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SchemaFunctions.java
@@ -68,7 +68,7 @@ public class SchemaFunctions {
// Janino cannot infer type
org.apache.drill.exec.record.MaterializedField materializedField =
(org.apache.drill.exec.record.MaterializedField) columns.get(columnName);
- org.apache.drill.common.types.TypeProtos.MajorType type = inputs[i + 1].getType(); // lgtm [java/index-out-of-bounds]
+ org.apache.drill.common.types.TypeProtos.MajorType type = inputs[i + 1].getType();
if (materializedField != null && !materializedField.getType().equals(type)) {
org.apache.drill.common.types.TypeProtos.MinorType leastRestrictiveType =
org.apache.drill.exec.resolver.TypeCastRules.getLeastRestrictiveType(
@@ -85,7 +85,7 @@ public class SchemaFunctions {
columns.put(columnName, clone);
} else {
if (type.getMinorType() == org.apache.drill.common.types.TypeProtos.MinorType.MAP) {
- columns.put(columnName, inputs[i + 1].getField()); // lgtm [java/index-out-of-bounds]
+ columns.put(columnName, inputs[i + 1].getField());
} else {
columns.put(columnName, org.apache.drill.exec.record.MaterializedField.create(columnName, type));
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/DeadBuf.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/DeadBuf.java
index dcaf2262be..c3c3539de5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/DeadBuf.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/DeadBuf.java
@@ -917,7 +917,7 @@ public class DeadBuf extends ByteBuf {
}
@Override
- public boolean equals(Object arg0) { //lgtm [java/unchecked-cast-in-equals]
+ public boolean equals(Object arg0) {
return false;
}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractPropertied.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractPropertied.java
index a04cdd9f4a..31bf83a690 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractPropertied.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractPropertied.java
@@ -120,7 +120,7 @@ public class AbstractPropertied implements Propertied {
}
@Override
- public boolean equals(Object o) { //lgtm [java/unchecked-cast-in-equals]
+ public boolean equals(Object o) {
if (o == this) {
return true;
}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
index 9af530f165..19ff04b12e 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
@@ -23,7 +23,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
public class JsonStringArrayList<E> extends ArrayList<E> {
-//lgtm [java/inconsistent-equals-and-hashcode]
+
private static ObjectMapper mapper;
static {
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
index 8536822197..337858fa6c 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
@@ -30,7 +30,7 @@ import com.fasterxml.jackson.datatype.joda.JodaModule;
* toString() method of the HashMap class to produce a JSON string instead
*/
public class JsonStringHashMap<K, V> extends LinkedHashMap<K, V> {
- // lgtm [java/inconsistent-equals-and-hashcode]
+
private static final ObjectMapper mapper = new ObjectMapper()
.registerModule(SerializationModule.getModule())
.registerModule(new JodaModule());
diff --git a/pom.xml b/pom.xml
index fb6081e672..cd134b1623 100644
--- a/pom.xml
+++ b/pom.xml
@@ -695,7 +695,6 @@
<exclude>**/clientlib/y2038/*.h</exclude> <!-- All the files here should have MIT License -->
<exclude>**/resources/parquet/**/*</exclude>
<exclude>**/.asf.yaml</exclude>
- <exclude>**/.lgtm.yml</exclude>
<exclude>**/*.woff2</exclude>
<exclude>**/*.ks</exclude>
<exclude>**/*.pcap</exclude>
[drill] 09/09: DRILL-8282: Update hadoop.dll and winutils.exe to 3.2.4. (#2635)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 3a236657758d659f5b961f69f98543a56ad8e272
Author: James Turton <91...@users.noreply.github.com>
AuthorDate: Mon Aug 29 16:35:55 2022 +0200
DRILL-8282: Update hadoop.dll and winutils.exe to 3.2.4. (#2635)
---
.github/workflows/ci.yml | 2 +-
.../src/main/resources/winutils/hadoop.dll | Bin 88576 -> 88576 bytes
.../src/main/resources/winutils/winutils.exe | Bin 118784 -> 118784 bytes
3 files changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0c7b24aeb5..185a113f85 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -29,7 +29,7 @@ jobs:
build:
name: Main Build
runs-on: ubuntu-latest
- timeout-minutes: 100
+ timeout-minutes: 120
strategy:
matrix:
# Java versions to run unit tests
diff --git a/distribution/src/main/resources/winutils/hadoop.dll b/distribution/src/main/resources/winutils/hadoop.dll
old mode 100644
new mode 100755
index 763c40acc4..c8f720fa37
Binary files a/distribution/src/main/resources/winutils/hadoop.dll and b/distribution/src/main/resources/winutils/hadoop.dll differ
diff --git a/distribution/src/main/resources/winutils/winutils.exe b/distribution/src/main/resources/winutils/winutils.exe
old mode 100644
new mode 100755
index b2c4819bf7..dfae152602
Binary files a/distribution/src/main/resources/winutils/winutils.exe and b/distribution/src/main/resources/winutils/winutils.exe differ
[drill] 04/09: DRILL-8280: Cannot ANALYZE files containing non-ASCII column names (#2625)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit e6f67d254191d51b5a1c5615dbeef92798ef04d2
Author: James Turton <91...@users.noreply.github.com>
AuthorDate: Thu Aug 18 02:12:44 2022 +0800
DRILL-8280: Cannot ANALYZE files containing non-ASCII column names (#2625)
---
.../drill/exec/expr/fn/impl/SchemaFunctions.java | 5 +-
.../drill/exec/sql/TestMetastoreCommands.java | 193 ++++++++++++---------
2 files changed, 116 insertions(+), 82 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SchemaFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SchemaFunctions.java
index 7bce81be08..a71447eb4a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SchemaFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SchemaFunctions.java
@@ -151,8 +151,9 @@ public class SchemaFunctions {
}
org.apache.drill.exec.record.metadata.TupleMetadata currentSchema =
- org.apache.drill.exec.expr.fn.impl.SchemaFunctions.getTupleMetadata(
- org.apache.drill.common.util.DrillStringUtils.toBinaryString(input.buffer, input.start, input.end));
+ org.apache.drill.exec.expr.fn.impl.SchemaFunctions.getTupleMetadata(
+ org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(input.start, input.end, input.buffer));
+
if (schemaHolder.obj == null) {
schemaHolder.obj = currentSchema;
return;
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestMetastoreCommands.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestMetastoreCommands.java
index 75f90868e4..7f42403c9f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestMetastoreCommands.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestMetastoreCommands.java
@@ -64,10 +64,12 @@ import java.nio.file.Paths;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -100,85 +102,87 @@ public class TestMetastoreCommands extends ClusterTest {
.build();
public static final Map<SchemaPath, ColumnStatistics<?>> TABLE_COLUMN_STATISTICS =
- ImmutableMap.<SchemaPath, ColumnStatistics<?>>builder()
- .put(SchemaPath.getSimplePath("o_shippriority"),
- getColumnStatistics(0, 0, 120L, TypeProtos.MinorType.INT))
- .put(SchemaPath.getSimplePath("o_orderstatus"),
- getColumnStatistics("F", "P", 120L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_orderpriority"),
- getColumnStatistics("1-URGENT", "5-LOW", 120L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_orderkey"),
- getColumnStatistics(1, 1319, 120L, TypeProtos.MinorType.INT))
- .put(SchemaPath.getSimplePath("o_clerk"),
- getColumnStatistics("Clerk#000000004", "Clerk#000000995", 120L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_totalprice"),
- getColumnStatistics(3266.69, 350110.21, 120L, TypeProtos.MinorType.FLOAT8))
- .put(SchemaPath.getSimplePath("o_comment"),
+ new LinkedHashMap<SchemaPath, ColumnStatistics<?>>()
+ {{
+ put(SchemaPath.getSimplePath("o_shippriority"),
+ getColumnStatistics(0, 0, 120L, TypeProtos.MinorType.INT));
+ put(SchemaPath.getSimplePath("o_orderstatus"),
+ getColumnStatistics("F", "P", 120L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_orderpriority"),
+ getColumnStatistics("1-URGENT", "5-LOW", 120L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_orderkey"),
+ getColumnStatistics(1, 1319, 120L, TypeProtos.MinorType.INT));
+ put(SchemaPath.getSimplePath("o_clerk"),
+ getColumnStatistics("Clerk#000000004", "Clerk#000000995", 120L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_totalprice"),
+ getColumnStatistics(3266.69, 350110.21, 120L, TypeProtos.MinorType.FLOAT8));
+ put(SchemaPath.getSimplePath("o_comment"),
getColumnStatistics(" about the final platelets. dependen",
- "zzle. carefully enticing deposits nag furio", 120L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_custkey"),
- getColumnStatistics(25, 1498, 120L, TypeProtos.MinorType.INT))
- .put(SchemaPath.getSimplePath("dir0"),
- getColumnStatistics("1994", "1996", 120L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("dir1"),
- getColumnStatistics("Q1", "Q4", 120L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_orderdate"),
- getColumnStatistics(757382400000L, 850953600000L, 120L, TypeProtos.MinorType.DATE))
- .build();
+ "zzle. carefully enticing deposits nag furio", 120L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_custkey"),
+ getColumnStatistics(25, 1498, 120L, TypeProtos.MinorType.INT));
+ put(SchemaPath.getSimplePath("dir0"),
+ getColumnStatistics("1994", "1996", 120L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("dir1"),
+ getColumnStatistics("Q1", "Q4", 120L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_orderdate"),
+ getColumnStatistics(757382400000L, 850953600000L, 120L, TypeProtos.MinorType.DATE));
+ }};
public static final Map<SchemaPath, ColumnStatistics<?>> DIR0_1994_SEGMENT_COLUMN_STATISTICS =
- ImmutableMap.<SchemaPath, ColumnStatistics<?>>builder()
- .put(SchemaPath.getSimplePath("o_shippriority"),
- getColumnStatistics(0, 0, 40L, TypeProtos.MinorType.INT))
- .put(SchemaPath.getSimplePath("o_orderstatus"),
- getColumnStatistics("F", "F", 40L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_orderpriority"),
- getColumnStatistics("1-URGENT", "5-LOW", 40L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_orderkey"),
- getColumnStatistics(5, 1031, 40L, TypeProtos.MinorType.INT))
- .put(SchemaPath.getSimplePath("o_clerk"),
- getColumnStatistics("Clerk#000000004", "Clerk#000000973", 40L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_totalprice"),
- getColumnStatistics(3266.69, 350110.21, 40L, TypeProtos.MinorType.FLOAT8))
- .put(SchemaPath.getSimplePath("o_comment"),
+ new LinkedHashMap<SchemaPath, ColumnStatistics<?>>()
+ {{
+ put(SchemaPath.getSimplePath("o_shippriority"),
+ getColumnStatistics(0, 0, 40L, TypeProtos.MinorType.INT));
+ put(SchemaPath.getSimplePath("o_orderstatus"),
+ getColumnStatistics("F", "F", 40L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_orderpriority"),
+ getColumnStatistics("1-URGENT", "5-LOW", 40L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_orderkey"),
+ getColumnStatistics(5, 1031, 40L, TypeProtos.MinorType.INT));
+ put(SchemaPath.getSimplePath("o_clerk"),
+ getColumnStatistics("Clerk#000000004", "Clerk#000000973", 40L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_totalprice"),
+ getColumnStatistics(3266.69, 350110.21, 40L, TypeProtos.MinorType.FLOAT8));
+ put(SchemaPath.getSimplePath("o_comment"),
getColumnStatistics(" accounts nag slyly. ironic, ironic accounts wake blithel",
- "yly final requests over the furiously regula", 40L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_custkey"),
- getColumnStatistics(25, 1469, 40L, TypeProtos.MinorType.INT))
- .put(SchemaPath.getSimplePath("dir0"),
- getColumnStatistics("1994", "1994", 40L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("dir1"),
- getColumnStatistics("Q1", "Q4", 40L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_orderdate"),
- getColumnStatistics(757382400000L, 788140800000L, 40L, TypeProtos.MinorType.DATE))
- .build();
+ "yly final requests over the furiously regula", 40L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_custkey"),
+ getColumnStatistics(25, 1469, 40L, TypeProtos.MinorType.INT));
+ put(SchemaPath.getSimplePath("dir0"),
+ getColumnStatistics("1994", "1994", 40L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("dir1"),
+ getColumnStatistics("Q1", "Q4", 40L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_orderdate"),
+ getColumnStatistics(757382400000L, 788140800000L, 40L, TypeProtos.MinorType.DATE));
+ }};
public static final Map<SchemaPath, ColumnStatistics<?>> DIR0_1994_Q1_SEGMENT_COLUMN_STATISTICS =
- ImmutableMap.<SchemaPath, ColumnStatistics<?>>builder()
- .put(SchemaPath.getSimplePath("o_shippriority"),
- getColumnStatistics(0, 0, 10L, TypeProtos.MinorType.INT))
- .put(SchemaPath.getSimplePath("o_orderstatus"),
- getColumnStatistics("F", "F", 10L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_orderpriority"),
- getColumnStatistics("1-URGENT", "5-LOW", 10L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_orderkey"),
- getColumnStatistics(66, 833, 10L, TypeProtos.MinorType.INT))
- .put(SchemaPath.getSimplePath("o_clerk"),
- getColumnStatistics("Clerk#000000062", "Clerk#000000973", 10L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_totalprice"),
- getColumnStatistics(3266.69, 132531.73, 10L, TypeProtos.MinorType.FLOAT8))
- .put(SchemaPath.getSimplePath("o_comment"),
+ new LinkedHashMap<SchemaPath, ColumnStatistics<?>>() {{
+ put(SchemaPath.getSimplePath("o_shippriority"),
+ getColumnStatistics(0, 0, 10L, TypeProtos.MinorType.INT));
+ put(SchemaPath.getSimplePath("o_orderstatus"),
+ getColumnStatistics("F", "F", 10L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_orderpriority"),
+ getColumnStatistics("1-URGENT", "5-LOW", 10L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_orderkey"),
+ getColumnStatistics(66, 833, 10L, TypeProtos.MinorType.INT));
+ put(SchemaPath.getSimplePath("o_clerk"),
+ getColumnStatistics("Clerk#000000062", "Clerk#000000973", 10L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_totalprice"),
+ getColumnStatistics(3266.69, 132531.73, 10L, TypeProtos.MinorType.FLOAT8));
+ put(SchemaPath.getSimplePath("o_comment"),
getColumnStatistics(" special pinto beans use quickly furiously even depende",
- "y pending requests integrate", 10L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_custkey"),
- getColumnStatistics(392, 1411, 10L, TypeProtos.MinorType.INT))
- .put(SchemaPath.getSimplePath("dir0"),
- getColumnStatistics("1994", "1994", 10L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("dir1"),
- getColumnStatistics("Q1", "Q1", 10L, TypeProtos.MinorType.VARCHAR))
- .put(SchemaPath.getSimplePath("o_orderdate"),
- getColumnStatistics(757382400000L, 764640000000L, 10L, TypeProtos.MinorType.DATE))
- .build();
+ "y pending requests integrate", 10L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_custkey"),
+ getColumnStatistics(392, 1411, 10L, TypeProtos.MinorType.INT));
+ put(SchemaPath.getSimplePath("dir0"),
+ getColumnStatistics("1994", "1994", 10L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("dir1"),
+ getColumnStatistics("Q1", "Q1", 10L, TypeProtos.MinorType.VARCHAR));
+ put(SchemaPath.getSimplePath("o_orderdate"),
+ getColumnStatistics(757382400000L, 764640000000L, 10L, TypeProtos.MinorType.DATE));
+ }};
public static final MetadataInfo TABLE_META_INFO = MetadataInfo.builder()
.type(MetadataType.TABLE)
@@ -3564,15 +3568,44 @@ public class TestMetastoreCommands extends ClusterTest {
}
}
- public static <T> ColumnStatistics<T> getColumnStatistics(T minValue, T maxValue,
- long rowCount, TypeProtos.MinorType minorType) {
+ @Test // DRILL-8280
+ public void testNonAsciiColumnName() throws Exception {
+ String tableName = "utf8_col_name";
+ String colName = "Käse";
+
+ run("create table dfs.tmp.%s as select 'Cheddar' as `%s`", tableName, colName);
+ try {
+ testBuilder()
+ .sqlQuery("analyze table dfs.tmp.`%s` refresh metadata", tableName)
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format("Collected / refreshed metadata for table [dfs.tmp.%s]", tableName))
+ .go();
+ String query = "select column_name from information_schema.`columns` where table_name='%s' and column_name='%s'";
+
+ testBuilder()
+ .sqlQuery(query, tableName, colName)
+ .unOrdered()
+ .baselineColumns("column_name")
+ .baselineValues(colName)
+ .go();
+ } finally {
+ run("analyze table dfs.tmp.`%s` drop metadata if exists", tableName);
+ run("drop table if exists dfs.tmp.`%s`", tableName);
+ }
+ }
+
+
+ public static <T> ColumnStatistics<T> getColumnStatistics(T minValue, T maxValue, long rowCount,
+ TypeProtos.MinorType minorType) {
return new ColumnStatistics<>(
- Arrays.asList(
- new StatisticsHolder<>(minValue, ColumnStatisticsKind.MIN_VALUE),
- new StatisticsHolder<>(maxValue, ColumnStatisticsKind.MAX_VALUE),
- new StatisticsHolder<>(rowCount, TableStatisticsKind.ROW_COUNT),
- new StatisticsHolder<>(rowCount, ColumnStatisticsKind.NON_NULL_VALUES_COUNT),
- new StatisticsHolder<>(0L, ColumnStatisticsKind.NULLS_COUNT)),
+ new ArrayList() {{
+ add(new StatisticsHolder<>(minValue, ColumnStatisticsKind.MIN_VALUE));
+ add(new StatisticsHolder<>(maxValue, ColumnStatisticsKind.MAX_VALUE));
+ add(new StatisticsHolder<>(rowCount, TableStatisticsKind.ROW_COUNT));
+ add(new StatisticsHolder<>(rowCount, ColumnStatisticsKind.NON_NULL_VALUES_COUNT));
+ add(new StatisticsHolder<>(0L, ColumnStatisticsKind.NULLS_COUNT));
+ }},
minorType);
}
[drill] 08/09: DRILL-8283: Add a configurable recursive file listing size limit (#2632)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 2a389176c6f0a8803f29cb2cf3c36835c337e2fc
Author: James Turton <91...@users.noreply.github.com>
AuthorDate: Thu Sep 8 07:12:19 2022 +0200
DRILL-8283: Add a configurable recursive file listing size limit (#2632)
---
.../src/main/resources/core-site-example.xml | 16 ++++-
.../org/apache/drill/exec/util/FileSystemUtil.java | 71 ++++++++++++++++++++--
.../apache/drill/exec/util/FileSystemUtilTest.java | 18 ++++++
3 files changed, 99 insertions(+), 6 deletions(-)
diff --git a/distribution/src/main/resources/core-site-example.xml b/distribution/src/main/resources/core-site-example.xml
index 0261658c01..d870172d76 100644
--- a/distribution/src/main/resources/core-site-example.xml
+++ b/distribution/src/main/resources/core-site-example.xml
@@ -172,5 +172,19 @@
</description>
</property>
-->
-
+ <!--
+ Use the next property to set a limit on the numer of files that Drill
+ will list by recursing into a DFS directory tree. When the limit is
+ encountered the initiating operation will fail with an error. The
+ intended application of this limit is to allow admins to protect their
+ Drillbits from an errant or malicious SELECT * FROM dfs.huge_workspace
+ LIMIT 10 query, which will cause an OOM given a big enough workspace of
+ files. Defaults to 0 which means no limit.
+ -->
+ <!--
+ <property>
+ <name>drill.exec.recursive_file_listing_max_size</name>
+ <value>10000</value>
+ </property>
+ --->
</configuration>
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/FileSystemUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/FileSystemUtil.java
index 82500da30f..488f3fb35a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/FileSystemUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/FileSystemUtil.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.util;
import org.apache.drill.common.exceptions.ErrorHelper;
+import org.apache.drill.common.exceptions.UserException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -28,9 +29,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinTask;
import java.util.concurrent.RecursiveTask;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -41,6 +44,7 @@ import java.util.stream.Stream;
public class FileSystemUtil {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemUtil.class);
+ public static final String RECURSIVE_FILE_LISTING_MAX_SIZE = "drill.exec.recursive_file_listing_max_size";
/**
* Filter that will accept all files and directories.
@@ -248,9 +252,24 @@ public class FileSystemUtil {
*/
private static List<FileStatus> listRecursive(FileSystem fs, Path path, Scope scope, boolean suppressExceptions, PathFilter filter) {
ForkJoinPool pool = new ForkJoinPool();
+ AtomicInteger fileCounter = new AtomicInteger(0);
+ int recursiveListingMaxSize = fs.getConf().getInt(RECURSIVE_FILE_LISTING_MAX_SIZE, 0);
+
try {
- RecursiveListing task = new RecursiveListing(fs, path, scope, suppressExceptions, filter);
+ RecursiveListing task = new RecursiveListing(
+ fs,
+ path,
+ scope,
+ suppressExceptions,
+ filter,
+ fileCounter,
+ recursiveListingMaxSize,
+ pool
+ );
return pool.invoke(task);
+ } catch (CancellationException ex) {
+ logger.debug("RecursiveListing task to list {} was cancelled.", path);
+ return Collections.<FileStatus>emptyList();
} finally {
pool.shutdown();
}
@@ -287,13 +306,29 @@ public class FileSystemUtil {
private final Scope scope;
private final boolean suppressExceptions;
private final PathFilter filter;
+ // Running count of files for comparison with RECURSIVE_FILE_LISTING_MAX_SIZE
+ private final AtomicInteger fileCounter;
+ private final int recursiveListingMaxSize;
+ private final ForkJoinPool pool;
- RecursiveListing(FileSystem fs, Path path, Scope scope, boolean suppressExceptions, PathFilter filter) {
+ RecursiveListing(
+ FileSystem fs,
+ Path path,
+ Scope scope,
+ boolean suppressExceptions,
+ PathFilter filter,
+ AtomicInteger fileCounter,
+ int recursiveListingMaxSize,
+ ForkJoinPool pool
+ ) {
this.fs = fs;
this.path = path;
this.scope = scope;
this.suppressExceptions = suppressExceptions;
this.filter = filter;
+ this.fileCounter = fileCounter;
+ this.recursiveListingMaxSize = recursiveListingMaxSize;
+ this.pool = pool;
}
@Override
@@ -302,12 +337,39 @@ public class FileSystemUtil {
List<RecursiveListing> tasks = new ArrayList<>();
try {
- for (FileStatus status : fs.listStatus(path, filter)) {
+ FileStatus[] dirFs = fs.listStatus(path, filter);
+ if (fileCounter.addAndGet(dirFs.length) > recursiveListingMaxSize && recursiveListingMaxSize > 0 ) {
+ try {
+ throw UserException
+ .resourceError()
+ .message(
+ "File listing size limit of %d exceeded recursing through path %s, see JVM system property %s",
+ recursiveListingMaxSize,
+ path,
+ RECURSIVE_FILE_LISTING_MAX_SIZE
+ )
+ .build(logger);
+ } finally {
+ // Attempt to abort all tasks
+ this.pool.shutdownNow();
+ }
+ }
+
+ for (FileStatus status : dirFs) {
if (isStatusApplicable(status, scope)) {
statuses.add(status);
}
if (status.isDirectory()) {
- RecursiveListing task = new RecursiveListing(fs, status.getPath(), scope, suppressExceptions, filter);
+ RecursiveListing task = new RecursiveListing(
+ fs,
+ status.getPath(),
+ scope,
+ suppressExceptions,
+ filter,
+ fileCounter,
+ recursiveListingMaxSize,
+ pool
+ );
task.fork();
tasks.add(task);
}
@@ -328,5 +390,4 @@ public class FileSystemUtil {
return statuses;
}
}
-
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTest.java
index 299f1cc65e..48ee441ab6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTest.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.util;
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
@@ -26,8 +28,10 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
+import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
public class FileSystemUtilTest extends FileSystemUtilTestBase {
@@ -199,4 +203,18 @@ public class FileSystemUtilTest extends FileSystemUtilTestBase {
assertTrue("Should return empty result", fileStatuses.isEmpty());
}
+ @Test // DRILL-8283
+ public void testRecursiveListingMaxSize() throws IOException {
+ Configuration conf = fs.getConf();
+ int oldSize = conf.getInt(FileSystemUtil.RECURSIVE_FILE_LISTING_MAX_SIZE, 0);
+ conf.setInt(FileSystemUtil.RECURSIVE_FILE_LISTING_MAX_SIZE, 5);
+
+ try {
+ FileSystemUtil.listAll(fs, new Path(base, "a"), true);
+ } catch (UserException ex) {
+ assertThat(ex.getMessage(), containsString("RESOURCE ERROR: File listing size limit"));
+ } finally {
+ conf.setInt(FileSystemUtil.RECURSIVE_FILE_LISTING_MAX_SIZE, oldSize);
+ }
+ }
}
[drill] 01/09: DRILL-8275: Prevent the JDBC Client from creating spurious paths in Zookeeper (#2617)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 64c4f25a1926c9f45ef5221710e8e760c05e0782
Author: luoc <lu...@apache.org>
AuthorDate: Mon Sep 5 13:48:11 2022 +0800
DRILL-8275: Prevent the JDBC Client from creating spurious paths in Zookeeper (#2617)
---
.../drill/exec/coord/zk/ZKClusterCoordinator.java | 36 +++++++++++++-----
.../org/apache/drill/exec/server/Drillbit.java | 3 +-
.../drill/exec/client/DrillClientStateTest.java | 43 ++++++++++++++++++++++
3 files changed, 71 insertions(+), 11 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
index aeb0dd0f20..108ca770c6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
@@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.ArrayList;
import java.util.Set;
import java.util.HashSet;
+import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -32,6 +33,7 @@ import java.util.regex.Pattern;
import org.apache.curator.framework.imps.DefaultACLProvider;
import org.apache.drill.shaded.guava.com.google.common.base.Throwables;
+import org.apache.zookeeper.data.Stat;
import org.apache.commons.collections.keyvalue.MultiKey;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
@@ -40,6 +42,7 @@ import org.apache.curator.framework.api.ACLProvider;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.framework.state.ConnectionStateListener;
import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.utils.ZKPaths;
import org.apache.curator.x.discovery.ServiceCache;
import org.apache.curator.x.discovery.ServiceDiscovery;
import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
@@ -78,15 +81,17 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
private ConcurrentHashMap<MultiKey, DrillbitEndpoint> endpointsMap = new ConcurrentHashMap<MultiKey,DrillbitEndpoint>();
private static final Pattern ZK_COMPLEX_STRING = Pattern.compile("(^.*?)/(.*)/([^/]*)$");
- public ZKClusterCoordinator(DrillConfig config, String connect) {
- this(config, connect, new DefaultACLProvider());
+ public ZKClusterCoordinator(DrillConfig config, String connect) throws Exception {
+ // As a Client, the namespace (aka zkRoot) should not be created.
+ this(config, connect, false, new DefaultACLProvider());
}
- public ZKClusterCoordinator(DrillConfig config, ACLProvider aclProvider) {
- this(config, null, aclProvider);
+ public ZKClusterCoordinator(DrillConfig config, ACLProvider aclProvider) throws Exception {
+ // As a Drillbit, it's required to create the zkRoot.
+ this(config, null, true, aclProvider);
}
- public ZKClusterCoordinator(DrillConfig config, String connect, ACLProvider aclProvider) {
+ public ZKClusterCoordinator(DrillConfig config, String connect, boolean createNamespace, ACLProvider aclProvider) throws Exception {
connect = connect == null || connect.isEmpty() ? config.getString(ExecConstants.ZK_CONNECTION) : connect;
String clusterId = config.getString(ExecConstants.SERVICE_NAME);
@@ -106,13 +111,23 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
RetryPolicy rp = new RetryNTimes(config.getInt(ExecConstants.ZK_RETRY_TIMES),
config.getInt(ExecConstants.ZK_RETRY_DELAY));
- curator = CuratorFrameworkFactory.builder()
- .namespace(zkRoot)
+
+ CuratorFrameworkFactory.Builder curatorBuilder = CuratorFrameworkFactory.builder()
.connectionTimeoutMs(config.getInt(ExecConstants.ZK_TIMEOUT))
.retryPolicy(rp)
.connectString(connect)
- .aclProvider(aclProvider)
- .build();
+ .aclProvider(aclProvider);
+
+ if (!createNamespace) { // Using ZK style in client, the root path should exist.
+ try (CuratorFramework client = curatorBuilder.build()) {
+ client.start();
+ Stat stat = client.checkExists().forPath(ZKPaths.PATH_SEPARATOR + zkRoot);
+ Objects.requireNonNull(stat, "The root path does not exist in the Zookeeper.");
+ }
+ }
+
+ curator = curatorBuilder.namespace(zkRoot).build();
+
curator.getConnectionStateListenable().addListener(new InitialConnectionListener());
curator.start();
discovery = newDiscovery();
@@ -219,6 +234,7 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
* triggered. State information is used during planning and
* initial client connection phases.
*/
+ @Override
public RegistrationHandle update(RegistrationHandle handle, State state) {
ZKRegistrationHandle h = (ZKRegistrationHandle) handle;
try {
@@ -348,7 +364,7 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
protected ServiceDiscovery<DrillbitEndpoint> newDiscovery() {
return ServiceDiscoveryBuilder
.builder(DrillbitEndpoint.class)
- .basePath("/")
+ .basePath(ZKPaths.PATH_SEPARATOR)
.client(curator)
.serializer(DrillServiceInstanceHelper.SERIALIZER)
.build();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
index 3bf20e04e7..b0cb47dde3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.server;
import org.apache.curator.framework.api.ACLProvider;
+import org.apache.curator.utils.ZKPaths;
import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.StackTrace;
import org.apache.drill.common.concurrent.ExtendedLatch;
@@ -179,7 +180,7 @@ public class Drillbit implements AutoCloseable {
} else {
String clusterId = config.getString(ExecConstants.SERVICE_NAME);
String zkRoot = config.getString(ExecConstants.ZK_ROOT);
- String drillClusterPath = "/" + zkRoot + "/" + clusterId;
+ String drillClusterPath = ZKPaths.PATH_SEPARATOR + zkRoot + ZKPaths.PATH_SEPARATOR + clusterId;
ACLProvider aclProvider = ZKACLProviderFactory.getACLProvider(config, drillClusterPath, context);
coord = new ZKClusterCoordinator(config, aclProvider);
storeProvider = new PersistentStoreRegistry<>(this.coord, config).newPStoreProvider();
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DrillClientStateTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DrillClientStateTest.java
new file mode 100644
index 0000000000..0a25e6e8d4
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DrillClientStateTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.client;
+
+import org.apache.curator.utils.ZKPaths;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.TestWithZookeeper;
+import org.apache.drill.exec.coord.zk.ZKClusterCoordinator;
+import org.junit.Test;
+
+public class DrillClientStateTest extends TestWithZookeeper {
+
+ @Test
+ public void testNotExistZkRoot() throws Exception {
+ // There is no drillbit startup, therefore the root path is not saved in ZK.
+ thrownException.expect(NullPointerException.class);
+ thrownException.expectMessage("root path does not exist");
+ DrillConfig config = DrillConfig.create();
+ String connString = zkHelper.getConnectionString();
+ String zkRoot = config.getString(ExecConstants.ZK_ROOT); // does not exist
+ connString = connString + ZKPaths.PATH_SEPARATOR + zkRoot;
+ try (ZKClusterCoordinator coordinator = new ZKClusterCoordinator(config, connString)) {
+ coordinator.start(10000);
+ }
+ }
+
+}
[drill] 06/09: DRILL-8282: Bump Hadoop-Common Version to 3.2.4 (CVE) (#2630)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 0d0708989edd086739563d439fdc7414b5eb076c
Author: PJ Fanning <pj...@users.noreply.github.com>
AuthorDate: Mon Aug 29 15:35:32 2022 +0100
DRILL-8282: Bump Hadoop-Common Version to 3.2.4 (CVE) (#2630)
---
contrib/storage-hbase/pom.xml | 12 +++++
contrib/storage-hive/core/pom.xml | 26 ++++++++++-
distribution/pom.xml | 4 ++
drill-yarn/pom.xml | 4 ++
exec/java-exec/pom.xml | 48 ++++++++++++++++++++
exec/rpc/pom.xml | 10 +++++
exec/vector/pom.xml | 10 +++++
logical/pom.xml | 8 ++++
metastore/iceberg-metastore/pom.xml | 12 +++++
metastore/metastore-api/pom.xml | 8 ++++
pom.xml | 90 +++++++++++++++++++++++++++++++++++--
11 files changed, 228 insertions(+), 4 deletions(-)
diff --git a/contrib/storage-hbase/pom.xml b/contrib/storage-hbase/pom.xml
index b2a8e51f8a..c3146cb0f4 100644
--- a/contrib/storage-hbase/pom.xml
+++ b/contrib/storage-hbase/pom.xml
@@ -181,6 +181,14 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
@@ -238,6 +246,10 @@
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml
index c024e014f4..e4342376b8 100644
--- a/contrib/storage-hive/core/pom.xml
+++ b/contrib/storage-hive/core/pom.xml
@@ -93,7 +93,15 @@
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
- </exclusion>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>hadoop-auth</artifactId>
<groupId>org.apache.hadoop</groupId>
@@ -147,6 +155,14 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -196,10 +212,18 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 2ef03c645a..f31bd05570 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -109,6 +109,10 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/drill-yarn/pom.xml b/drill-yarn/pom.xml
index 08ddf56f92..e0586c3f20 100644
--- a/drill-yarn/pom.xml
+++ b/drill-yarn/pom.xml
@@ -88,6 +88,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>slf4j-reload4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
</exclusions>
</dependency>
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index c436590be8..9d0c46ac7b 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -70,6 +70,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>slf4j-reload4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -389,6 +393,14 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -409,6 +421,14 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
@@ -439,6 +459,14 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -459,6 +487,10 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -486,6 +518,10 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -636,6 +672,14 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
@@ -717,6 +761,10 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/exec/rpc/pom.xml b/exec/rpc/pom.xml
index dc0606f381..2aae609f34 100644
--- a/exec/rpc/pom.xml
+++ b/exec/rpc/pom.xml
@@ -61,6 +61,16 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml
index 81fa911f48..02a376a54a 100644
--- a/exec/vector/pom.xml
+++ b/exec/vector/pom.xml
@@ -65,6 +65,16 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
diff --git a/logical/pom.xml b/logical/pom.xml
index 719ec10204..38847af986 100644
--- a/logical/pom.xml
+++ b/logical/pom.xml
@@ -93,6 +93,14 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/metastore/iceberg-metastore/pom.xml b/metastore/iceberg-metastore/pom.xml
index 394a054ff3..a1b6b9891f 100644
--- a/metastore/iceberg-metastore/pom.xml
+++ b/metastore/iceberg-metastore/pom.xml
@@ -97,10 +97,18 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
@@ -174,6 +182,10 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/metastore/metastore-api/pom.xml b/metastore/metastore-api/pom.xml
index b633af3a7e..e6cd30184c 100644
--- a/metastore/metastore-api/pom.xml
+++ b/metastore/metastore-api/pom.xml
@@ -58,6 +58,14 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/pom.xml b/pom.xml
index 537890e5e6..fb6081e672 100644
--- a/pom.xml
+++ b/pom.xml
@@ -77,7 +77,7 @@
<curator.version>5.2.0</curator.version>
<wiremock.standalone.version>2.23.2</wiremock.standalone.version>
<jmockit.version>1.47</jmockit.version>
- <logback.version>1.2.9</logback.version>
+ <logback.version>1.2.11</logback.version>
<mockito.version>3.11.2</mockito.version>
<!--
Currently, Hive storage plugin only supports Apache Hive 3.1.2 or vendor specific variants of the
@@ -85,7 +85,7 @@
for example parquet-hadoop-bundle and derby dependencies.
-->
<hive.version>3.1.2</hive.version>
- <hadoop.version>3.2.3</hadoop.version>
+ <hadoop.version>3.2.4</hadoop.version>
<hbase.version>2.4.9</hbase.version>
<fmpp.version>1.0</fmpp.version>
<freemarker.version>2.3.28</freemarker.version>
@@ -1440,6 +1440,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>slf4j-reload4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
<exclusion>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
@@ -1488,6 +1492,14 @@
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -1511,6 +1523,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>slf4j-reload4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging-api</artifactId>
@@ -1538,6 +1554,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
@@ -1597,6 +1617,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -1657,6 +1681,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
@@ -1708,6 +1736,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
@@ -1853,6 +1885,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
@@ -2115,6 +2151,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>mockito-all</artifactId>
<groupId>org.mockito</groupId>
@@ -2278,6 +2318,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>mockito-all</artifactId>
<groupId>org.mockito</groupId>
@@ -2449,6 +2493,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>mockito-all</artifactId>
<groupId>org.mockito</groupId>
@@ -2577,7 +2625,7 @@
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
- <artifactId>slf4j-log4j12</artifactId>
+ <artifactId>slf4j-reload4j</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
@@ -2730,6 +2778,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
@@ -2775,6 +2827,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>asm</artifactId>
<groupId>asm</groupId>
@@ -2938,6 +2994,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>slf4j-reload4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
<exclusion>
<artifactId>mockito-all</artifactId>
<groupId>org.mockito</groupId>
@@ -3013,6 +3073,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>mockito-all</artifactId>
<groupId>org.mockito</groupId>
@@ -3143,6 +3207,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
@@ -3239,6 +3307,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
@@ -3314,6 +3386,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
@@ -3359,6 +3435,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>asm</artifactId>
<groupId>asm</groupId>
@@ -3879,6 +3959,10 @@
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
<exclusion>
<artifactId>reload4j</artifactId>
<groupId>ch.qos.reload4j</groupId>