[impala] 03/05: IMPALA-9287: Add support for embedded HMS in CDP builds

[jo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 55612f99910414b01627c537203737d8f24971bf
Author: skyyws <sk...@163.com>
AuthorDate: Thu Jan 16 17:53:56 2020 +0800

    IMPALA-9287: Add support for embedded HMS in CDP builds
    
    In some situations, an embedded HMS is enough for catalogd server.
    And we've already implemented this in IMPALA-8974. But after
    setting USE_CDP_HIVE=true and rebuilt impala, the custom cluster
    test case test_kudu_table_create_without_hms would failed due to
    lacking of related jars. The solution is to add related maven
    dependency in $IMPALA_HOME/fe/pom.xml and
    $IMPALA_HOME/shaded-deps/pom.xml.
    
    Tests:
      * Ran test_kudu_table_create_without_hms.py by setting
      USE_CDP_HIVE=true locally
    
    Change-Id: Ibc7d7e30cd560d43bb707dec54f4494355809f66
    Reviewed-on: http://gerrit.cloudera.org:8080/15057
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 bin/bootstrap_system.sh                            |  6 +++++
 fe/pom.xml                                         | 29 +++++++++++++++++++---
 fe/src/test/resources/hive-site.xml.py             |  4 +++
 .../test_kudu_table_create_without_hms.py          |  1 -
 4 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh
index c6ef1d1..fca2ad6 100755
--- a/bin/bootstrap_system.sh
+++ b/bin/bootstrap_system.sh
@@ -312,6 +312,12 @@ redhat sudo sed -ri 's/local +all +all +ident/local all all trust/g' \
 # Accept md5 passwords from localhost
 redhat sudo sed -i -e 's,\(host.*\)ident,\1md5,' /var/lib/pgsql/data/pg_hba.conf
 
+# Modfiy pg max connections to 500 for IMPALA-9287
+ubuntu sudo sed -i 's/\(max_connections = \)\S*/\1500/g' \
+  /etc/postgresql/*/main/postgresql.conf
+redhat sudo sed -i 's/\(max_connections = \)\S*/\1500/g' \
+  /var/lib/pgsql/data/postgresql.conf
+
 ubuntu sudo service postgresql start
 redhat6 sudo service postgresql start
 redhat7 notindocker sudo service postgresql start
diff --git a/fe/pom.xml b/fe/pom.xml
index f861096..da1fdb8 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -1226,13 +1226,12 @@ under the License.
             </exclusion>
           </exclusions>
         </dependency>
-        <!-- Needed by tests like TestCaseLoader instantiate HMS in embedded mode which
-        needs datanucleus as test dependency-->
+        <!-- IMPALA-9287: Needed when catalogd used embedded HMS-->
         <dependency>
           <groupId>org.datanucleus</groupId>
           <artifactId>javax.jdo</artifactId>
           <version>3.2.0-m3</version>
-          <scope>test</scope>
+          <scope>runtime</scope>
         </dependency>
         <!-- IMPALA-8766: Include Knox jars on the classpath -->
         <dependency>
@@ -1247,6 +1246,30 @@ under the License.
             </exclusion>
           </exclusions>
         </dependency>
+
+        <!-- IMPALA-9287: Needed when catalogd used embedded HMS-->
+        <dependency>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-metastore</artifactId>
+          <version>${hive.version}</version>
+          <scope>runtime</scope>
+          <exclusions>
+            <!-- Impala uses log4j v1; avoid pulling in slf4j handling for log4j2 -->
+            <exclusion>
+              <groupId>org.apache.logging.log4j</groupId>
+              <artifactId>log4j-slf4j-impl</artifactId>
+            </exclusion>
+            <!-- https://issues.apache.org/jira/browse/HADOOP-14903 -->
+            <exclusion>
+              <groupId>net.minidev</groupId>
+              <artifactId>json-smart</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.hadoop</groupId>
+              <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+            </exclusion>
+          </exclusions>
+        </dependency>
       </dependencies>
     </profile>
 
diff --git a/fe/src/test/resources/hive-site.xml.py b/fe/src/test/resources/hive-site.xml.py
index 1f891fe..eb68401 100644
--- a/fe/src/test/resources/hive-site.xml.py
+++ b/fe/src/test/resources/hive-site.xml.py
@@ -155,6 +155,10 @@ CONFIG.update({
 
 if variant == 'without_hms_config':
   CONFIG.clear()
+  if hive_major_version >= 3:
+    CONFIG.update({
+      'metastore.expression.proxy': 'org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy'
+    })
 
 # Database and JDO-related configs:
 db_type = os.environ.get('HMS_DB_TYPE', 'postgres')
diff --git a/tests/custom_cluster/test_kudu_table_create_without_hms.py b/tests/custom_cluster/test_kudu_table_create_without_hms.py
index cfe957e..acbd3b1 100644
--- a/tests/custom_cluster/test_kudu_table_create_without_hms.py
+++ b/tests/custom_cluster/test_kudu_table_create_without_hms.py
@@ -29,7 +29,6 @@ TBL_NAME = "test_kudu_table_create_without_hms"
 class TestCreatingKuduTableWithoutHMS(CustomClusterTestSuite):
   """Test creating kudu managed table without hms"""
 
-  @SkipIfHive3.without_hms_not_supported
   @pytest.mark.execute_serially
   @CustomClusterTestSuite.with_args(hive_conf_dir=HIVE_SITE_WITHOUT_HMS_DIR)
   def test_kudu_table_create_without_hms(self, unique_database):