You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mg...@apache.org on 2019/10/16 08:09:56 UTC
[hive] branch master updated: HIVE-22340 Prevent shaded imports
(Miklos Gergely, reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
mgergely pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5ce0b40 HIVE-22340 Prevent shaded imports (Miklos Gergely, reviewed by Zoltan Haindrich)
5ce0b40 is described below
commit 5ce0b4008ed1d6b7bdce0d037cc6cf7010bd948b
Author: miklosgergely <mg...@cloudera.com>
AuthorDate: Tue Oct 15 01:50:01 2019 +0200
HIVE-22340 Prevent shaded imports (Miklos Gergely, reviewed by Zoltan Haindrich)
---
.../hive/ql/txn/compactor/TestCompactor.java | 3 +-
.../org/apache/hadoop/hive/kudu/KuduTestSetup.java | 3 +-
.../hadoop/hive/kudu/TestKuduInputFormat.java | 2 +-
.../hadoop/hive/kudu/TestKuduOutputFormat.java | 3 +-
.../hadoop/hive/kudu/TestKuduPredicateHandler.java | 2 +-
.../org/apache/hadoop/hive/kudu/TestKuduSerDe.java | 3 +-
.../hive/llap/metrics/ReadWriteLockMetrics.java | 3 +-
pom.xml | 27 ++++++++++++++++
.../ql/optimizer/signature/RelTreeSignature.java | 6 +---
.../org/apache/hadoop/hive/ql/TestTxnCommands.java | 3 +-
.../hadoop/hive/ql/TestTxnCommandsForMmTable.java | 36 ----------------------
.../hive/ql/stats/TestStatsUpdaterThread.java | 3 +-
.../hadoop/hive/ql/stats/TestStatsUtils.java | 3 +-
.../generic/TestGenericUDAFBinarySetFunctions.java | 3 +-
.../hadoop/hive/metastore/utils/FileUtils.java | 3 +-
15 files changed, 48 insertions(+), 55 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index 5b8fb4b..f2ea30b 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -38,7 +38,6 @@ import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.curator.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -100,6 +99,8 @@ import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
public class TestCompactor {
private static final AtomicInteger salt = new AtomicInteger(new Random().nextInt());
private static final Logger LOG = LoggerFactory.getLogger(TestCompactor.class);
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/kudu/KuduTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/kudu/KuduTestSetup.java
index 2a0f04c..85ab1eb 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/kudu/KuduTestSetup.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/kudu/KuduTestSetup.java
@@ -26,9 +26,10 @@ import org.apache.kudu.Type;
import org.apache.kudu.client.CreateTableOptions;
import org.apache.kudu.client.KuduClient;
import org.apache.kudu.client.KuduException;
-import org.apache.kudu.shaded.com.google.common.collect.ImmutableList;
import org.apache.kudu.test.cluster.MiniKuduCluster;
+import com.google.common.collect.ImmutableList;
+
import java.io.File;
import java.util.Arrays;
diff --git a/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduInputFormat.java b/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduInputFormat.java
index feb6f75..653cdc2 100644
--- a/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduInputFormat.java
+++ b/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduInputFormat.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.kudu;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.kudu.KuduInputFormat.KuduInputSplit;
@@ -42,7 +43,6 @@ import org.apache.kudu.client.KuduSession;
import org.apache.kudu.client.KuduTable;
import org.apache.kudu.client.PartialRow;
import org.apache.kudu.client.RowResult;
-import org.apache.kudu.shaded.com.google.common.collect.ImmutableList;
import org.apache.kudu.test.KuduTestHarness;
import org.junit.Before;
import org.junit.Rule;
diff --git a/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduOutputFormat.java b/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduOutputFormat.java
index 8a1cf26..c208e38 100644
--- a/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduOutputFormat.java
+++ b/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduOutputFormat.java
@@ -28,12 +28,13 @@ import org.apache.kudu.client.KuduScanner;
import org.apache.kudu.client.KuduTable;
import org.apache.kudu.client.PartialRow;
import org.apache.kudu.client.RowResult;
-import org.apache.kudu.shaded.com.google.common.collect.ImmutableList;
import org.apache.kudu.test.KuduTestHarness;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import com.google.common.collect.ImmutableList;
+
import java.math.BigDecimal;
import java.sql.Timestamp;
import java.util.ArrayList;
diff --git a/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduPredicateHandler.java b/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduPredicateHandler.java
index ad09c5b..20f17dc 100644
--- a/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduPredicateHandler.java
+++ b/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduPredicateHandler.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.kudu;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
@@ -51,7 +52,6 @@ import org.apache.kudu.client.KuduPredicate;
import org.apache.kudu.client.KuduScanner;
import org.apache.kudu.client.KuduTable;
import org.apache.kudu.client.PartialRow;
-import org.apache.kudu.shaded.com.google.common.collect.ImmutableList;
import org.apache.kudu.test.KuduTestHarness;
import org.junit.Before;
import org.junit.Rule;
diff --git a/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduSerDe.java b/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduSerDe.java
index da2a989..589e0d9 100644
--- a/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduSerDe.java
+++ b/kudu-handler/src/test/org/apache/hadoop/hive/kudu/TestKuduSerDe.java
@@ -28,12 +28,13 @@ import org.apache.kudu.Schema;
import org.apache.kudu.Type;
import org.apache.kudu.client.CreateTableOptions;
import org.apache.kudu.client.PartialRow;
-import org.apache.kudu.shaded.com.google.common.collect.ImmutableList;
import org.apache.kudu.test.KuduTestHarness;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import com.google.common.collect.ImmutableList;
+
import java.math.BigDecimal;
import java.sql.Timestamp;
import java.util.Arrays;
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/metrics/ReadWriteLockMetrics.java b/llap-common/src/java/org/apache/hadoop/hive/llap/metrics/ReadWriteLockMetrics.java
index 7d52a15..189c684 100644
--- a/llap-common/src/java/org/apache/hadoop/hive/llap/metrics/ReadWriteLockMetrics.java
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/metrics/ReadWriteLockMetrics.java
@@ -18,8 +18,7 @@
package org.apache.hadoop.hive.llap.metrics;
-import avro.shaded.com.google.common.annotations.VisibleForTesting;
-
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.io.Serializable;
diff --git a/pom.xml b/pom.xml
index 47634f4..ff2c86a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1178,6 +1178,13 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>de.skuzzle.enforcer</groupId>
+ <artifactId>restrict-imports-enforcer-rule</artifactId>
+ <version>0.9.0</version>
+ </dependency>
+ </dependencies>
<executions>
<execution>
<id>enforce-no-snapshots</id>
@@ -1233,6 +1240,26 @@
<fail>true</fail>
</configuration>
</execution>
+ <execution>
+ <id>check-banned-imports</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <restrictImports implementation="de.skuzzle.enforcer.restrictimports.RestrictImports">
+ <reason>Do not use shaded imports</reason>
+ <bannedImports>
+ <bannedImport>**.shaded.**</bannedImport>
+ <bannedImport>org.spark_project.guava.**</bannedImport>
+ <bannedImport>jersey.repackaged.com.google.**</bannedImport>
+ </bannedImports>
+ <includeTestCode>true</includeTestCode>
+ </restrictImports>
+ </rules>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/RelTreeSignature.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/RelTreeSignature.java
index 40e93f0..536c334 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/RelTreeSignature.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/RelTreeSignature.java
@@ -25,17 +25,13 @@ import java.util.List;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.externalize.RelWriterImpl;
import org.apache.calcite.sql.SqlExplainLevel;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelJsonImpl;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelWriterImplCopy;
import com.fasterxml.jackson.annotation.JsonIdentityInfo;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.ObjectIdGenerators;
-
-import avro.shaded.com.google.common.base.Objects;
+import com.google.common.base.Objects;
/**
* Operator tree signature.
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 900290e..1da01be 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -34,7 +34,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import org.apache.curator.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -76,6 +75,8 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
/**
* The LockManager is not ready, but for no-concurrency straight-line path we can
* test AC=true, and AC=false with commit/rollback/exception and test resulting data.
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java
index a067dc9..d4c9121 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java
@@ -19,54 +19,18 @@
package org.apache.hadoop.hive.ql;
import java.io.File;
-import java.lang.reflect.Field;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.curator.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
-import org.apache.hadoop.hive.metastore.api.CompactionRequest;
-import org.apache.hadoop.hive.metastore.api.CompactionType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
-import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
-import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
-import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
-import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
-import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService;
import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.apache.hadoop.hive.metastore.txn.TxnStore;
-import org.apache.hadoop.hive.metastore.txn.TxnUtils;
-import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
-import org.apache.hadoop.hive.ql.io.BucketCodec;
-import org.apache.hadoop.hive.ql.io.HiveInputFormat;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService;
-import org.apache.hadoop.hive.ql.txn.compactor.Cleaner;
-import org.apache.hadoop.hive.ql.txn.compactor.Initiator;
-import org.apache.hadoop.hive.ql.txn.compactor.Worker;
-import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
index 19d1928..84827d1 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
@@ -26,7 +26,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.curator.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.ValidWriteIdList;
@@ -53,6 +52,8 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
public class TestStatsUpdaterThread {
@SuppressWarnings("unused")
static final private Logger LOG = LoggerFactory.getLogger(TestStatsUpdaterThread.class);
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java
index 4add290..850db9f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.plan.ColStatistics.Range;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.junit.Test;
-import org.spark_project.guava.collect.Sets;
+
+import com.google.common.collect.Sets;
public class TestStatsUtils {
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDAFBinarySetFunctions.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDAFBinarySetFunctions.java
index 3d33ea1..2762f18 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDAFBinarySetFunctions.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDAFBinarySetFunctions.java
@@ -32,13 +32,12 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuf
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.LongWritable;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import jersey.repackaged.com.google.common.collect.Lists;
+import com.google.common.collect.Lists;
@RunWith(Parameterized.class)
public class TestGenericUDAFBinarySetFunctions {
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
index 155ecb1..bf206ff 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.metastore.utils;
-import org.apache.curator.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
@@ -34,6 +33,8 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;