You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/09/08 01:52:37 UTC
[17/38] hive git commit: HIVE-14693 : Some paritions will be left out
when partition number is the multiple of the option
hive.msck.repair.batch.size (Ferdinand Xu via Thejas Nair)
HIVE-14693 : Some paritions will be left out when partition number is the multiple of the option hive.msck.repair.batch.size (Ferdinand Xu via Thejas Nair)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1d926ae8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1d926ae8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1d926ae8
Branch: refs/heads/hive-14535
Commit: 1d926ae8180857033704610fe065415893660ee8
Parents: 8d7dbd4
Author: Ferdinand Xu <ch...@intel.com>
Authored: Fri Sep 2 21:09:33 2016 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Fri Sep 2 21:10:23 2016 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java | 3 +--
ql/src/test/queries/clientpositive/msck_repair_batchsize.q | 5 ++++-
ql/src/test/results/clientpositive/msck_repair_batchsize.q.out | 5 +++--
3 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1d926ae8/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index c5d526d..b19ac49 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -1877,10 +1877,9 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
apd.addPartition(Warehouse.makeSpecFromName(part.getPartitionName()), null);
repairOutput.add("Repair: Added partition to metastore " + msckDesc.getTableName()
+ ':' + part.getPartitionName());
- if (counter == batch_size) {
+ if (counter % batch_size == 0 || counter == partsNotInMs.size()) {
db.createPartitions(apd);
apd = new AddPartitionDesc(table.getDbName(), table.getTableName(), false);
- counter = 0;
}
}
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/1d926ae8/ql/src/test/queries/clientpositive/msck_repair_batchsize.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/msck_repair_batchsize.q b/ql/src/test/queries/clientpositive/msck_repair_batchsize.q
index 60970e2..06e4507 100644
--- a/ql/src/test/queries/clientpositive/msck_repair_batchsize.q
+++ b/ql/src/test/queries/clientpositive/msck_repair_batchsize.q
@@ -1,4 +1,4 @@
-set hive.msck.repair.batch.size=1;
+set hive.msck.repair.batch.size=2;
DROP TABLE IF EXISTS repairtable;
@@ -8,7 +8,10 @@ MSCK TABLE repairtable;
dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=a/p2=a;
dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=b/p2=a;
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=c/p2=a;
+dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=a/p2=a/datafile;
dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=b/p2=a/datafile;
+dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/datafile;
MSCK TABLE default.repairtable;
http://git-wip-us.apache.org/repos/asf/hive/blob/1d926ae8/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out b/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out
index 86ac031..0300f20 100644
--- a/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out
+++ b/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out
@@ -18,14 +18,15 @@ PREHOOK: query: MSCK TABLE default.repairtable
PREHOOK: type: MSCK
POSTHOOK: query: MSCK TABLE default.repairtable
POSTHOOK: type: MSCK
-Partitions not in metastore: repairtable:p1=a/p2=a repairtable:p1=b/p2=a
+Partitions not in metastore: repairtable:p1=a/p2=a repairtable:p1=b/p2=a repairtable:p1=c/p2=a
PREHOOK: query: MSCK REPAIR TABLE default.repairtable
PREHOOK: type: MSCK
POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
POSTHOOK: type: MSCK
-Partitions not in metastore: repairtable:p1=a/p2=a repairtable:p1=b/p2=a
+Partitions not in metastore: repairtable:p1=a/p2=a repairtable:p1=b/p2=a repairtable:p1=c/p2=a
Repair: Added partition to metastore default.repairtable:p1=a/p2=a
Repair: Added partition to metastore default.repairtable:p1=b/p2=a
+Repair: Added partition to metastore default.repairtable:p1=c/p2=a
PREHOOK: query: MSCK TABLE repairtable
PREHOOK: type: MSCK
POSTHOOK: query: MSCK TABLE repairtable