You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2016/08/25 21:13:12 UTC
hive git commit: Revert "HIVE-14462 : Reduce number of partition
check calls in add_partitions (Rajesh Balamohan via Sergey Shelukhin)"
Repository: hive
Updated Branches:
refs/heads/master 3d325d7ba -> aff5d9c59
Revert "HIVE-14462 : Reduce number of partition check calls in add_partitions (Rajesh Balamohan via Sergey Shelukhin)"
This reverts commit 9343fee5d10ab5ab64692d9723a6c35e77adefc3.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aff5d9c5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aff5d9c5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aff5d9c5
Branch: refs/heads/master
Commit: aff5d9c590f3bf5d4e4485fb3efc3c4a9794c5dc
Parents: 3d325d7
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Thu Aug 25 14:12:36 2016 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Aug 25 14:12:53 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/HiveMetaStore.java | 18 ++-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 6 +-
.../hadoop/hive/ql/metadata/CheckResult.java | 28 ++---
.../hive/ql/metadata/HiveMetaStoreChecker.java | 50 ++-------
.../ql/metadata/TestHiveMetaStoreChecker.java | 110 +++++++++----------
5 files changed, 91 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/aff5d9c5/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index f4391b3..3f85ca6 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -2311,8 +2311,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
+ dbName + "." + tblName + ": " + part);
}
- MetaStoreUtils.validatePartitionNameCharacters(part.getValues(),
- partitionValidationPattern);
+ boolean shouldAdd = startAddPartition(ms, part, ifNotExists);
+ if (!shouldAdd) {
+ existingParts.add(part);
+ LOG.info("Not adding partition " + part + " as it already exists");
+ continue;
+ }
+
partFutures.add(threadPool.submit(new Callable() {
@Override
@@ -2471,10 +2476,11 @@ public class HiveMetaStore extends ThriftHiveMetastore {
throw new MetaException("Partition does not belong to target table "
+ dbName + "." + tblName + ": " + part);
}
-
- MetaStoreUtils.validatePartitionNameCharacters(part.getValues(),
- partitionValidationPattern);
-
+ boolean shouldAdd = startAddPartition(ms, part, ifNotExists);
+ if (!shouldAdd) {
+ LOG.info("Not adding partition " + part + " as it already exists");
+ continue;
+ }
partFutures.add(threadPool.submit(new Callable() {
@Override public Object call() throws Exception {
boolean madeDir = createLocationForAddedPartition(table, part);
http://git-wip-us.apache.org/repos/asf/hive/blob/aff5d9c5/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index ca459a9..a59b781 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -1766,7 +1766,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
private void msckAddPartitionsOneByOne(Hive db, Table table,
- Set<CheckResult.PartitionResult> partsNotInMs, List<String> repairOutput) {
+ List<CheckResult.PartitionResult> partsNotInMs, List<String> repairOutput) {
for (CheckResult.PartitionResult part : partsNotInMs) {
try {
db.createPartition(table, Warehouse.makeSpecFromName(part.getPartitionName()));
@@ -1825,7 +1825,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
HiveMetaStoreChecker checker = new HiveMetaStoreChecker(db);
String[] names = Utilities.getDbTableName(msckDesc.getTableName());
checker.checkMetastore(names[0], names[1], msckDesc.getPartSpecs(), result);
- Set<CheckResult.PartitionResult> partsNotInMs = result.getPartitionsNotInMs();
+ List<CheckResult.PartitionResult> partsNotInMs = result.getPartitionsNotInMs();
if (msckDesc.isRepairPartitions() && !partsNotInMs.isEmpty()) {
AbstractList<String> vals = null;
String settingStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_MSCK_PATH_VALIDATION);
@@ -1957,7 +1957,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
* @throws IOException
* In case the writing fails
*/
- private boolean writeMsckResult(Set<? extends Object> result, String msg,
+ private boolean writeMsckResult(List<? extends Object> result, String msg,
Writer out, boolean wrote) throws IOException {
if (!result.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/aff5d9c5/ql/src/java/org/apache/hadoop/hive/ql/metadata/CheckResult.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/CheckResult.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CheckResult.java
index 36b9250..ec9deeb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/CheckResult.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CheckResult.java
@@ -17,23 +17,23 @@
*/
package org.apache.hadoop.hive.ql.metadata;
-import java.util.Set;
-import java.util.TreeSet;
+import java.util.ArrayList;
+import java.util.List;
/**
* Result class used by the HiveMetaStoreChecker.
*/
public class CheckResult {
- private Set<String> tablesNotOnFs = new TreeSet<String>();
- private Set<String> tablesNotInMs = new TreeSet<String>();
- private Set<PartitionResult> partitionsNotOnFs = new TreeSet<PartitionResult>();
- private Set<PartitionResult> partitionsNotInMs = new TreeSet<PartitionResult>();
+ private List<String> tablesNotOnFs = new ArrayList<String>();
+ private List<String> tablesNotInMs = new ArrayList<String>();
+ private List<PartitionResult> partitionsNotOnFs = new ArrayList<PartitionResult>();
+ private List<PartitionResult> partitionsNotInMs = new ArrayList<PartitionResult>();
/**
* @return a list of tables not found on the filesystem.
*/
- public Set<String> getTablesNotOnFs() {
+ public List<String> getTablesNotOnFs() {
return tablesNotOnFs;
}
@@ -41,14 +41,14 @@ public class CheckResult {
* @param tablesNotOnFs
* a list of tables not found on the filesystem.
*/
- public void setTablesNotOnFs(Set<String> tablesNotOnFs) {
+ public void setTablesNotOnFs(List<String> tablesNotOnFs) {
this.tablesNotOnFs = tablesNotOnFs;
}
/**
* @return a list of tables not found in the metastore.
*/
- public Set<String> getTablesNotInMs() {
+ public List<String> getTablesNotInMs() {
return tablesNotInMs;
}
@@ -56,14 +56,14 @@ public class CheckResult {
* @param tablesNotInMs
* a list of tables not found in the metastore.
*/
- public void setTablesNotInMs(Set<String> tablesNotInMs) {
+ public void setTablesNotInMs(List<String> tablesNotInMs) {
this.tablesNotInMs = tablesNotInMs;
}
/**
* @return a list of partitions not found on the fs
*/
- public Set<PartitionResult> getPartitionsNotOnFs() {
+ public List<PartitionResult> getPartitionsNotOnFs() {
return partitionsNotOnFs;
}
@@ -71,14 +71,14 @@ public class CheckResult {
* @param partitionsNotOnFs
* a list of partitions not found on the fs
*/
- public void setPartitionsNotOnFs(Set<PartitionResult> partitionsNotOnFs) {
+ public void setPartitionsNotOnFs(List<PartitionResult> partitionsNotOnFs) {
this.partitionsNotOnFs = partitionsNotOnFs;
}
/**
* @return a list of partitions not found in the metastore
*/
- public Set<PartitionResult> getPartitionsNotInMs() {
+ public List<PartitionResult> getPartitionsNotInMs() {
return partitionsNotInMs;
}
@@ -86,7 +86,7 @@ public class CheckResult {
* @param partitionsNotInMs
* a list of partitions not found in the metastore
*/
- public void setPartitionsNotInMs(Set<PartitionResult> partitionsNotInMs) {
+ public void setPartitionsNotInMs(List<PartitionResult> partitionsNotInMs) {
this.partitionsNotInMs = partitionsNotInMs;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/aff5d9c5/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
index d7ad4e3..34b76b8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.metadata;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@@ -32,8 +33,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
-import com.google.common.collect.Sets;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileStatus;
@@ -114,10 +113,10 @@ public class HiveMetaStoreChecker {
// check the specified partitions
checkTable(dbName, tableName, partitions, result);
}
- LOG.info("Number of partitionsNotInMs=" + result.getPartitionsNotInMs()
- + ", partitionsNotOnFs=" + result.getPartitionsNotOnFs()
- + ", tablesNotInMs=" + result.getTablesNotInMs()
- + ", tablesNotOnFs=" + result.getTablesNotOnFs());
+ Collections.sort(result.getPartitionsNotInMs());
+ Collections.sort(result.getPartitionsNotOnFs());
+ Collections.sort(result.getTablesNotInMs());
+ Collections.sort(result.getTablesNotOnFs());
} catch (MetaException e) {
throw new HiveException(e);
} catch (TException e) {
@@ -318,17 +317,11 @@ public class HiveMetaStoreChecker {
// remove the partition paths we know about
allPartDirs.removeAll(partPaths);
- Set<String> partColNames = Sets.newHashSet();
- for(FieldSchema fSchema : table.getPartCols()) {
- partColNames.add(fSchema.getName());
- }
-
// we should now only have the unexpected folders left
for (Path partPath : allPartDirs) {
FileSystem fs = partPath.getFileSystem(conf);
String partitionName = getPartitionName(fs.makeQualified(tablePath),
- partPath, partColNames);
- LOG.debug("PartitionName: " + partitionName);
+ partPath);
if (partitionName != null) {
PartitionResult pr = new PartitionResult();
@@ -338,7 +331,6 @@ public class HiveMetaStoreChecker {
result.getPartitionsNotInMs().add(pr);
}
}
- LOG.debug("Number of partitions not in metastore : " + result.getPartitionsNotInMs().size());
}
/**
@@ -348,37 +340,19 @@ public class HiveMetaStoreChecker {
* Path of the table.
* @param partitionPath
* Path of the partition.
- * @param partCols
- * Set of partition columns from table definition
* @return Partition name, for example partitiondate=2008-01-01
*/
- static String getPartitionName(Path tablePath, Path partitionPath,
- Set<String> partCols) {
+ private String getPartitionName(Path tablePath, Path partitionPath) {
String result = null;
Path currPath = partitionPath;
- LOG.debug("tablePath:" + tablePath + ", partCols: " + partCols);
-
while (currPath != null && !tablePath.equals(currPath)) {
- // format: partition=p_val
- // Add only when table partition colName matches
- String[] parts = currPath.getName().split("=");
- if (parts != null && parts.length > 0) {
- if (parts.length != 2) {
- LOG.warn(currPath.getName() + " is not a valid partition name");
- return result;
- }
-
- String partitionName = parts[0];
- if (partCols.contains(partitionName)) {
- if (result == null) {
- result = currPath.getName();
- } else {
- result = currPath.getName() + Path.SEPARATOR + result;
- }
- }
+ if (result == null) {
+ result = currPath.getName();
+ } else {
+ result = currPath.getName() + Path.SEPARATOR + result;
}
+
currPath = currPath.getParent();
- LOG.debug("currPath=" + currPath);
}
return result;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/aff5d9c5/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
index cda6e30..3f26bcd 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
@@ -24,7 +24,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Lists;
import junit.framework.TestCase;
import org.apache.hadoop.fs.FileSystem;
@@ -112,19 +111,19 @@ public class TestHiveMetaStoreChecker extends TestCase {
CheckResult result = new CheckResult();
checker.checkMetastore(dbName, null, null, result);
// we haven't added anything so should return an all ok
- assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// check table only, should not exist in ms
result = new CheckResult();
checker.checkMetastore(dbName, tableName, null, result);
assertEquals(1, result.getTablesNotInMs().size());
- assertEquals(tableName, result.getTablesNotInMs().iterator().next());
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs());
+ assertEquals(tableName, result.getTablesNotInMs().get(0));
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
Database db = new Database();
db.setName(dbName);
@@ -140,18 +139,18 @@ public class TestHiveMetaStoreChecker extends TestCase {
// first check all (1) tables
result = new CheckResult();
checker.checkMetastore(dbName, null, null, result);
- assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// then let's check the one we know about
result = new CheckResult();
checker.checkMetastore(dbName, tableName, null, result);
- assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// remove the table folder
fs = table.getPath().getFileSystem(hive.getConf());
@@ -160,11 +159,11 @@ public class TestHiveMetaStoreChecker extends TestCase {
// now this shouldn't find the path on the fs
result = new CheckResult();
checker.checkMetastore(dbName, tableName, null, result);
- assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());;
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());;
assertEquals(1, result.getTablesNotOnFs().size());
- assertEquals(tableName, result.getTablesNotOnFs().iterator().next());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs());
+ assertEquals(tableName, result.getTablesNotOnFs().get(0));
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// put it back and one additional table
fs.mkdirs(table.getPath());
@@ -177,10 +176,10 @@ public class TestHiveMetaStoreChecker extends TestCase {
result = new CheckResult();
checker.checkMetastore(dbName, null, null, result);
assertEquals(1, result.getTablesNotInMs().size());
- assertEquals(fakeTable.getName(), Lists.newArrayList(result.getTablesNotInMs()).get(0));
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs());
+ assertEquals(fakeTable.getName(), result.getTablesNotInMs().get(0));
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// create a new external table
hive.dropTable(dbName, tableName);
@@ -190,10 +189,10 @@ public class TestHiveMetaStoreChecker extends TestCase {
// should return all ok
result = new CheckResult();
checker.checkMetastore(dbName, null, null, result);
- assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
}
public void testPartitionsCheck() throws HiveException, MetaException,
@@ -219,26 +218,13 @@ public class TestHiveMetaStoreChecker extends TestCase {
CheckResult result = new CheckResult();
checker.checkMetastore(dbName, tableName, null, result);
// all is well
- assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
List<Partition> partitions = hive.getPartitions(table);
assertEquals(2, partitions.size());
- // add a fake partition dir on fs to ensure that it does not get added
- fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf());
- Path fakePart = new Path(table.getDataLocation().toString(),
- "fakedate=2009-01-01/fakecity=sanjose");
- fs.mkdirs(fakePart);
- fs.deleteOnExit(fakePart);
- checker.checkMetastore(dbName, tableName, null, result);
- assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
- assertEquals(0, result.getPartitionsNotOnFs().size());
- assertEquals(0, result.getPartitionsNotInMs().size());
- assertEquals(2, partitions.size()); //no additional partitions got added
-
Partition partToRemove = partitions.get(0);
// As this partition (partdate=2008-01-01/partcity=london) is the only
// partition under (partdate=2008-01-01)
@@ -250,24 +236,27 @@ public class TestHiveMetaStoreChecker extends TestCase {
result = new CheckResult();
checker.checkMetastore(dbName, tableName, null, result);
// missing one partition on fs
- assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
assertEquals(1, result.getPartitionsNotOnFs().size());
- assertEquals(partToRemove.getName(), result.getPartitionsNotOnFs().iterator().next()
+ assertEquals(partToRemove.getName(), result.getPartitionsNotOnFs().get(0)
.getPartitionName());
- assertEquals(partToRemove.getTable().getTableName(),
- result.getPartitionsNotOnFs().iterator().next().getTableName());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs());
+ assertEquals(partToRemove.getTable().getTableName(), result
+ .getPartitionsNotOnFs().get(0).getTableName());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
List<Map<String, String>> partsCopy = new ArrayList<Map<String, String>>();
partsCopy.add(partitions.get(1).getSpec());
// check only the partition that exists, all should be well
result = new CheckResult();
checker.checkMetastore(dbName, tableName, partsCopy, result);
- assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
+
+ // put the other one back
+ fs.mkdirs(partToRemovePath);
// old test is moved to msck_repair_2.q
@@ -276,11 +265,12 @@ public class TestHiveMetaStoreChecker extends TestCase {
hive.createTable(table);
result = new CheckResult();
checker.checkMetastore(dbName, null, null, result);
- assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
- assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotOnFs());
- assertEquals(Collections.<String>emptySet(), result.getPartitionsNotInMs()); //--0e
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs()); //--0e
System.err.println("Test completed - partition check");
+
}
public void testDataDeletion() throws HiveException, MetaException,