You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2018/01/11 21:21:47 UTC
[1/2] hive git commit: HIVE-18379 - ALTER TABLE authorization_part
SET PROPERTIES (PARTITIONL_LEVEL_PRIVILEGE=TRUE);
fails when authorization_part is MicroManaged table. (Steve Yeom, via Eugene
Koifman)
Repository: hive
Updated Branches:
refs/heads/master 103c5a4f5 -> 42a2e11a1
HIVE-18379 - ALTER TABLE authorization_part SET PROPERTIES (PARTITIONL_LEVEL_PRIVILEGE=TRUE); fails when authorization_part is MicroManaged table. (Steve Yeom, via Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da1c2154
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da1c2154
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da1c2154
Branch: refs/heads/master
Commit: da1c21544d7e47f4d78e04449bacc481701d7930
Parents: 103c5a4
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Thu Jan 11 13:02:45 2018 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Thu Jan 11 13:02:45 2018 -0800
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/da1c2154/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index bb105fe..fe80d60 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -1448,11 +1448,16 @@ public class AcidUtils {
// in many cases the conversion might be illegal.
// The only thing we allow is tx = true w/o tx-props, for backward compat.
String transactional = props.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+ String transactionalProp = props.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+
+ if (transactional == null && transactionalProp == null) {
+ // Not affected or the op is not about transactional.
+ return null;
+ }
+
if(transactional == null) {
transactional = tbl.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
}
- String transactionalProp = props.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
- if (transactional == null && transactionalProp == null) return null; // Not affected.
boolean isSetToTxn = "true".equalsIgnoreCase(transactional);
if (transactionalProp == null) {
if (isSetToTxn) return false; // Assume the full ACID table.
[2/2] hive git commit: HIVE-18190 - Consider looking at ORC file
schema rather than using _metadata_acid file (Eugene Koifman,
reviewed by Alan Gates)
Posted by ek...@apache.org.
HIVE-18190 - Consider looking at ORC file schema rather than using _metadata_acid file (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42a2e11a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42a2e11a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42a2e11a
Branch: refs/heads/master
Commit: 42a2e11a1518a2797add821e53442c16f9f390b5
Parents: da1c215
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Thu Jan 11 13:21:34 2018 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Thu Jan 11 13:21:34 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 52 +++++++++++++++++++-
.../apache/hadoop/hive/ql/metadata/Hive.java | 1 -
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 2 +-
.../apache/hadoop/hive/ql/io/TestAcidUtils.java | 15 ++++--
4 files changed, 61 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a2e11a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index fe80d60..fb361f3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -43,13 +43,17 @@ import org.apache.hadoop.hive.metastore.api.DataOperationType;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.io.orc.OrcFile;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater;
+import org.apache.hadoop.hive.ql.io.orc.Reader;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hive.common.util.Ref;
+import org.apache.orc.FileFormatException;
import org.apache.orc.impl.OrcAcidUtils;
import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
@@ -124,7 +128,7 @@ public class AcidUtils {
public static final Pattern LEGACY_BUCKET_DIGIT_PATTERN = Pattern.compile("^[0-9]{6}");
/**
* A write into a non-aicd table produces files like 0000_0 or 0000_0_copy_1
- * (Unless via Load Data statment)
+ * (Unless via Load Data statement)
*/
public static final PathFilter originalBucketFilter = new PathFilter() {
@Override
@@ -1525,7 +1529,8 @@ public class AcidUtils {
throw ioe;
}
}
- public static boolean isRawFormat(Path baseOrDeltaDir, FileSystem fs) throws IOException {
+ //should be useful for import/export
+ public static boolean isImport(Path baseOrDeltaDir, FileSystem fs) throws IOException {
Path formatFile = new Path(baseOrDeltaDir, METADATA_FILE);
if(!fs.exists(formatFile)) {
return false;
@@ -1554,5 +1559,48 @@ public class AcidUtils {
throw e;
}
}
+
+ /**
+ * Chooses 1 representantive file from {@code baseOrDeltaDir}
+ * This assumes that all files in the dir are of the same type: either written by an acid
+ * write or Load Data. This should always be the case for an Acid table.
+ */
+ private static Path chooseFile(Path baseOrDeltaDir, FileSystem fs) throws IOException {
+ if(!(baseOrDeltaDir.getName().startsWith(BASE_PREFIX) ||
+ baseOrDeltaDir.getName().startsWith(DELTA_PREFIX))) {
+ throw new IllegalArgumentException(baseOrDeltaDir + " is not a base/delta");
+ }
+ FileStatus[] dataFiles = fs.listStatus(new Path[] {baseOrDeltaDir}, originalBucketFilter);
+ return dataFiles != null && dataFiles.length > 0 ? dataFiles[0].getPath() : null;
+ }
+
+ /**
+ * Checks if the files in base/delta dir are a result of Load Data statement and thus do not
+ * have ROW_IDs embedded in the data.
+ * @param baseOrDeltaDir base or delta file.
+ */
+ public static boolean isRawFormat(Path baseOrDeltaDir, FileSystem fs) throws IOException {
+ Path dataFile = chooseFile(baseOrDeltaDir, fs);
+ if (dataFile == null) {
+ //directory is empty or doesn't have any that could have been produced by load data
+ return false;
+ }
+ try {
+ Reader reader = OrcFile.createReader(dataFile, OrcFile.readerOptions(fs.getConf()));
+ /*
+ acid file would have schema like <op, otid, writerId, rowid, ctid, <f1, ... fn>> so could
+ check it this way once/if OrcRecordUpdater.ACID_KEY_INDEX_NAME is removed
+ TypeDescription schema = reader.getSchema();
+ List<String> columns = schema.getFieldNames();
+ */
+ return OrcInputFormat.isOriginal(reader);
+ } catch (FileFormatException ex) {
+ //We may be parsing a delta for Insert-only table which may not even be an ORC file so
+ //cannot have ROW_IDs in it.
+ LOG.debug("isRawFormat() called on " + dataFile + " which is not an ORC file: " +
+ ex.getMessage());
+ return true;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/42a2e11a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 20d7593..41614d4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1921,7 +1921,6 @@ public class Hive {
if(!FileUtils.mkdir(fs, destPath, conf)) {
LOG.warn(destPath + " already exists?!?!");
}
- AcidUtils.MetaDataFile.createMetaFile(destPath, fs, true);
} catch (IOException e) {
throw new HiveException("load: error while creating " + destPath + ";loadFileType=" + loadFileType, e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/42a2e11a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
index 075aac5..9d6b408 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
@@ -866,7 +866,7 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
addPartitionColsToSelect(targetTable.getPartCols(), rewrittenQueryStr, target);
rewrittenQueryStr.append(" HAVING count(*) > 1");
- //say table T has partiton p, we are generating
+ //say table T has partition p, we are generating
//select cardinality_violation(ROW_ID, p) WHERE ... GROUP BY ROW__ID, p
//the Group By args are passed to cardinality_violation to add the violating value to the error msg
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/42a2e11a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
index 76618ff..0c309c0 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
@@ -105,9 +105,14 @@ public class TestAcidUtils {
@Test
public void testParsing() throws Exception {
+ Configuration conf = new Configuration();
+ MockFileSystem fs = new MockFileSystem(conf,
+ //new MockFile("mock:/tmp/base_000123/bucket_00001", 500, new byte[0]),
+ new MockFile("mock:/tmp/delta_000005_000006/bucket_00001", 500, new byte[0]),
+ new MockFile("mock:/tmp/delete_delta_000005_000006/bucket_00001", 500,
+ new byte[0]));
assertEquals(123, AcidUtils.parseBase(new Path("/tmp/base_000123")));
Path dir = new Path("/tmp/tbl");
- Configuration conf = new Configuration();
AcidOutputFormat.Options opts =
AcidUtils.parseBaseOrDeltaBucketFilename(new Path(dir, "base_567/bucket_123"),
conf);
@@ -116,15 +121,15 @@ public class TestAcidUtils {
assertEquals(567, opts.getMaximumTransactionId());
assertEquals(0, opts.getMinimumTransactionId());
assertEquals(123, opts.getBucketId());
- opts = AcidUtils.parseBaseOrDeltaBucketFilename(new Path(dir, "delta_000005_000006/bucket_00001"),
- conf);
+ opts = AcidUtils.parseBaseOrDeltaBucketFilename(
+ new MockPath(fs, dir + "/delta_000005_000006/bucket_00001"), conf);
assertEquals(false, opts.getOldStyle());
assertEquals(false, opts.isWritingBase());
assertEquals(6, opts.getMaximumTransactionId());
assertEquals(5, opts.getMinimumTransactionId());
assertEquals(1, opts.getBucketId());
- opts = AcidUtils.parseBaseOrDeltaBucketFilename(new Path(dir, "delete_delta_000005_000006/bucket_00001"),
- conf);
+ opts = AcidUtils.parseBaseOrDeltaBucketFilename(
+ new MockPath(fs, dir + "/delete_delta_000005_000006/bucket_00001"), conf);
assertEquals(false, opts.getOldStyle());
assertEquals(false, opts.isWritingBase());
assertEquals(6, opts.getMaximumTransactionId());