[hive] branch master updated: HIVE-21963: TransactionalValidationListener.validateTableStructure should check the partition directories in the case of partitioned tables (Jason Dere, reviewed by Vaibhav Gumashta)

[jd...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jdere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 65cdce1  HIVE-21963: TransactionalValidationListener.validateTableStructure should check the partition directories in the case of partitioned tables (Jason Dere, reviewed by Vaibhav Gumashta)
65cdce1 is described below

commit 65cdce16fe40ef93ecdda763ead67ed130b3dab5
Author: Jason Dere <jd...@cloudera.com>
AuthorDate: Tue Jul 16 09:42:59 2019 -0700

    HIVE-21963: TransactionalValidationListener.validateTableStructure should check the partition directories in the case of partitioned tables (Jason Dere, reviewed by Vaibhav Gumashta)
---
 .../metastore/TransactionalValidationListener.java | 59 ++++++++++++++++++++--
 1 file changed, 54 insertions(+), 5 deletions(-)

diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index afa6e4c..b1a92ef 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.metastore;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
@@ -28,11 +29,12 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.InitializeTableWriteIdsRequest;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
@@ -455,16 +457,63 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
    */
   private void validateTableStructure(IHMSHandler hmsHandler, Table table)
     throws MetaException {
-    Path tablePath;
+    Warehouse wh = hmsHandler.getWh();
+    if (isPartitionedTable(table)) {
+      // Validate each partition directory
+      List<Partition> partitions = getTablePartitions(hmsHandler, table);
+      for (Partition partition : partitions) {
+        Path partPath = wh.getDnsPath(new Path(partition.getSd().getLocation()));
+        validateTableStructureForPath(hmsHandler, wh, table, partPath);
+      }
+    } else {
+      // Non-partitioned - only need to worry about the base table directory
+      Path tablePath = getTablePath(hmsHandler, wh, table);
+      validateTableStructureForPath(hmsHandler, wh, table, tablePath);
+    }
+  }
+
+  private List<Partition> getTablePartitions(IHMSHandler hmsHandler, Table table) throws MetaException {
+    try {
+      RawStore rawStore = hmsHandler.getMS();
+      String catName = getTableCatalog(table);
+      List<Partition> partitions = rawStore.getPartitions(catName, table.getDbName(), table.getTableName(), -1);
+      return partitions;
+    } catch (Exception err) {
+      String msg = "Error getting partitions for " + Warehouse.getQualifiedName(table);
+      LOG.error(msg, err);
+      MetaException e1 = new MetaException(msg);
+      e1.initCause(err);
+      throw e1;
+    }
+  }
+
+  private Path getTablePath(IHMSHandler hmsHandler, Warehouse wh, Table table) throws MetaException {
+    Path tablePath = null;
     try {
-      Warehouse wh = hmsHandler.getWh();
       if (table.getSd().getLocation() == null || table.getSd().getLocation().isEmpty()) {
         String catName = getTableCatalog(table);
         tablePath = wh.getDefaultTablePath(hmsHandler.getMS().getDatabase(
-            catName, table.getDbName()), table);
+                catName, table.getDbName()), table);
       } else {
         tablePath = wh.getDnsPath(new Path(table.getSd().getLocation()));
       }
+    } catch (Exception err) {
+      MetaException e1 = new MetaException("Error getting table path for " + Warehouse.getQualifiedName(table));
+      e1.initCause(err);
+    }
+    return tablePath;
+  }
+
+  private static boolean isPartitionedTable(Table tableObj) {
+    List<FieldSchema> partKeys = tableObj.getPartitionKeys();
+    if (partKeys != null && partKeys.size() > 0) {
+      return true;
+    }
+    return false;
+  }
+
+  private void validateTableStructureForPath(IHMSHandler hmsHandler, Warehouse wh, Table table, Path tablePath) throws MetaException {
+    try {
       FileSystem fs = wh.getFs(tablePath);
       //FileSystem fs = FileSystem.get(getConf());
       FileUtils.RemoteIteratorWithFilter iterator =
@@ -485,7 +534,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
             + fileStatus.getPath());
         }
       }
-    } catch (IOException|NoSuchObjectException e) {
+    } catch (IOException e) {
       String msg = "Unable to list files for " + Warehouse.getQualifiedName(table);
       LOG.error(msg, e);
       MetaException e1 = new MetaException(msg);