You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/09/11 23:02:11 UTC
svn commit: r1624389 [1/3] - in /hive/branches/cbo: ./
common/src/java/org/apache/hadoop/hive/conf/
itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/
itests/hiv...
Author: gunther
Date: Thu Sep 11 21:02:09 2014
New Revision: 1624389
URL: http://svn.apache.org/r1624389
Log:
Merge latest trunk into cbo branch. (Gunther Hagleitner)
Added:
hive/branches/cbo/ql/src/test/results/clientpositive/tez/parallel.q.out
- copied unchanged from r1624386, hive/trunk/ql/src/test/results/clientpositive/tez/parallel.q.out
Modified:
hive/branches/cbo/ (props changed)
hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/cbo/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java
hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedClientSideAuthorizationProvider.java
hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHiveServer2.java
hive/branches/cbo/itests/src/test/resources/testconfiguration.properties
hive/branches/cbo/metastore/if/hive_metastore.thrift
hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java
hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java
hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java
hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponseElement.java
hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java
hive/branches/cbo/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
hive/branches/cbo/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/AppMasterEventProcessor.java
hive/branches/cbo/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q
hive/branches/cbo/ql/src/test/queries/clientpositive/groupby1.q
hive/branches/cbo/ql/src/test/queries/clientpositive/groupby4.q
hive/branches/cbo/ql/src/test/queries/clientpositive/having.q
hive/branches/cbo/ql/src/test/queries/clientpositive/limit_pushdown.q
hive/branches/cbo/ql/src/test/queries/clientpositive/merge1.q
hive/branches/cbo/ql/src/test/queries/clientpositive/merge2.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_19.q
hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out
hive/branches/cbo/ql/src/test/resources/orc-file-dump.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_merge_orc.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/groupby1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/groupby4.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/having.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/limit_pushdown.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/merge1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/merge2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_analyze.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge3.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge4.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/groupby1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/having.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/merge1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/merge2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_analyze.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_merge3.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_merge4.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_19.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vectorized_ptf.q.out
hive/branches/cbo/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
hive/branches/cbo/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
hive/branches/cbo/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java
hive/branches/cbo/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/ssh/TestSSHCommandExecutor.java
Propchange: hive/branches/cbo/
------------------------------------------------------------------------------
Merged /hive/trunk:r1624135-1624386
Modified: hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Sep 11 21:02:09 2014
@@ -862,6 +862,10 @@ public class HiveConf extends Configurat
"only affect the light weight encoding for integers. This flag will not\n" +
"change the compression level of higher level compression codec (like ZLIB)."),
+ HIVE_ORC_COMPRESSION_STRATEGY("hive.exec.orc.compression.strategy", "SPEED", new StringSet("SPEED", "COMPRESSION"),
+ "Define the compression strategy to use while writing data. \n" +
+ "This changes the compression level of higher level compression codec (like ZLIB)."),
+
HIVE_ORC_INCLUDE_FILE_FOOTER_IN_SPLITS("hive.orc.splits.include.file.footer", false,
"If turned on splits generated by orc will include metadata about the stripes in the file. This\n" +
"data is read remotely (from the client or HS2 machine) and sent to all the tasks."),
@@ -1796,7 +1800,7 @@ public class HiveConf extends Configurat
"vertices to the tez application master. These events will be used to prune unnecessary partitions."),
TEZ_DYNAMIC_PARTITION_PRUNING_MAX_EVENT_SIZE("hive.tez.dynamic.partition.pruning.max.event.size", 1*1024*1024L,
"Maximum size of events sent by processors in dynamic pruning. If this size is crossed no pruning will take place."),
- TEZ_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE("hive.tez.dynamic.parition.pruning.max.data.size", 100*1024*1024L,
+ TEZ_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE("hive.tez.dynamic.partition.pruning.max.data.size", 100*1024*1024L,
"Maximum total data size of events in dynamic pruning.")
;
Modified: hive/branches/cbo/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java (original)
+++ hive/branches/cbo/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java Thu Sep 11 21:02:09 2014
@@ -154,11 +154,6 @@ public class TestStorageBasedMetastoreAu
allowWriteAccessViaAcl(userName, warehouseDir.toString());
}
- protected void disallowCreateDatabase(String userName)
- throws Exception {
- disallowWriteAccessViaAcl(userName, warehouseDir.toString());
- }
-
@Override
protected void allowCreateInDb(String dbName, String userName, String location)
throws Exception {
Modified: hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedClientSideAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedClientSideAuthorizationProvider.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedClientSideAuthorizationProvider.java (original)
+++ hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedClientSideAuthorizationProvider.java Thu Sep 11 21:02:09 2014
@@ -85,7 +85,7 @@ public class TestStorageBasedClientSideA
protected void assertNoPrivileges(CommandProcessorResponse ret){
assertNotNull(ret);
assertFalse(0 == ret.getResponseCode());
- assertTrue(ret.getErrorMessage().indexOf("not permitted") != -1);
+ assertTrue(ret.getErrorMessage().indexOf("AccessControlException") != -1);
}
Modified: hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHiveServer2.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHiveServer2.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHiveServer2.java (original)
+++ hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHiveServer2.java Thu Sep 11 21:02:09 2014
@@ -18,9 +18,9 @@
package org.apache.hive.jdbc.miniHS2;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -31,7 +31,7 @@ import org.apache.hive.service.cli.Opera
import org.apache.hive.service.cli.RowSet;
import org.apache.hive.service.cli.SessionHandle;
import org.junit.After;
-import org.junit.Assert;
+import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -39,44 +39,60 @@ import org.junit.Test;
public class TestHiveServer2 {
private static MiniHS2 miniHS2 = null;
- private Map<String, String> confOverlay;
+ private static Map<String, String> confOverlay;
@BeforeClass
public static void beforeTest() throws Exception {
miniHS2 = new MiniHS2(new HiveConf());
+ confOverlay = new HashMap<String, String>();
+ confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ miniHS2.start(confOverlay);
+ }
+
+ @AfterClass
+ public static void afterTest() throws Exception {
+ miniHS2.stop();
}
@Before
public void setUp() throws Exception {
- confOverlay = new HashMap<String, String>();
- confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
- miniHS2.start(confOverlay);
}
@After
public void tearDown() throws Exception {
- miniHS2.stop();
}
+ /**
+ * Open a new session and run a test query
+ * @throws Exception
+ */
@Test
public void testConnection() throws Exception {
- String tabName = "testTab1";
+ String tableName = "TestHiveServer2TestConnection";
CLIServiceClient serviceClient = miniHS2.getServiceClient();
SessionHandle sessHandle = serviceClient.openSession("foo", "bar");
- serviceClient.executeStatement(sessHandle, "DROP TABLE IF EXISTS tab", confOverlay);
- serviceClient.executeStatement(sessHandle, "CREATE TABLE " + tabName + " (id INT)", confOverlay);
+ serviceClient.executeStatement(sessHandle, "DROP TABLE IF EXISTS " + tableName, confOverlay);
+ serviceClient.executeStatement(sessHandle, "CREATE TABLE " + tableName + " (id INT)", confOverlay);
OperationHandle opHandle = serviceClient.executeStatement(sessHandle, "SHOW TABLES", confOverlay);
RowSet rowSet = serviceClient.fetchResults(opHandle);
assertFalse(rowSet.numRows() == 0);
+ serviceClient.executeStatement(sessHandle, "DROP TABLE IF EXISTS " + tableName, confOverlay);
+ serviceClient.closeSession(sessHandle);
}
+
+ /**
+ * Open a new session and execute a set command
+ * @throws Exception
+ */
@Test
public void testGetVariableValue() throws Exception {
CLIServiceClient serviceClient = miniHS2.getServiceClient();
SessionHandle sessHandle = serviceClient.openSession("foo", "bar");
OperationHandle opHandle = serviceClient.executeStatement(sessHandle, "set system:os.name", confOverlay);
RowSet rowSet = serviceClient.fetchResults(opHandle);
- Assert.assertEquals(1, rowSet.numRows());
+ assertEquals(1, rowSet.numRows());
+ serviceClient.closeSession(sessHandle);
}
}
Modified: hive/branches/cbo/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/src/test/resources/testconfiguration.properties?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/cbo/itests/src/test/resources/testconfiguration.properties Thu Sep 11 21:02:09 2014
@@ -96,6 +96,7 @@ minitez.query.files.shared=alter_merge_2
orc_merge2.q,\
orc_merge3.q,\
orc_merge4.q,\
+ parallel.q,\
ptf.q,\
sample1.q,\
script_env_var1.q,\
Modified: hive/branches/cbo/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/if/hive_metastore.thrift?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/metastore/if/hive_metastore.thrift (original)
+++ hive/branches/cbo/metastore/if/hive_metastore.thrift Thu Sep 11 21:02:09 2014
@@ -630,12 +630,12 @@ struct ShowCompactRequest {
struct ShowCompactResponseElement {
1: required string dbname,
2: required string tablename,
- 3: required string partitionname,
+ 3: optional string partitionname,
4: required CompactionType type,
5: required string state,
- 6: required string workerid,
- 7: required i64 start,
- 8: required string runAs,
+ 6: optional string workerid,
+ 7: optional i64 start,
+ 8: optional string runAs,
}
struct ShowCompactResponse {
Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp Thu Sep 11 21:02:09 2014
@@ -9027,8 +9027,8 @@ void swap(ShowCompactRequest &a, ShowCom
(void) b;
}
-const char* ShowCompactResponseElement::ascii_fingerprint = "42464F3A97707F984FDE462104223A69";
-const uint8_t ShowCompactResponseElement::binary_fingerprint[16] = {0x42,0x46,0x4F,0x3A,0x97,0x70,0x7F,0x98,0x4F,0xDE,0x46,0x21,0x04,0x22,0x3A,0x69};
+const char* ShowCompactResponseElement::ascii_fingerprint = "2F338C265DC4FD82DD13F4966FE43F13";
+const uint8_t ShowCompactResponseElement::binary_fingerprint[16] = {0x2F,0x33,0x8C,0x26,0x5D,0xC4,0xFD,0x82,0xDD,0x13,0xF4,0x96,0x6F,0xE4,0x3F,0x13};
uint32_t ShowCompactResponseElement::read(::apache::thrift::protocol::TProtocol* iprot) {
@@ -9043,12 +9043,8 @@ uint32_t ShowCompactResponseElement::rea
bool isset_dbname = false;
bool isset_tablename = false;
- bool isset_partitionname = false;
bool isset_type = false;
bool isset_state = false;
- bool isset_workerid = false;
- bool isset_start = false;
- bool isset_runAs = false;
while (true)
{
@@ -9077,7 +9073,7 @@ uint32_t ShowCompactResponseElement::rea
case 3:
if (ftype == ::apache::thrift::protocol::T_STRING) {
xfer += iprot->readString(this->partitionname);
- isset_partitionname = true;
+ this->__isset.partitionname = true;
} else {
xfer += iprot->skip(ftype);
}
@@ -9103,7 +9099,7 @@ uint32_t ShowCompactResponseElement::rea
case 6:
if (ftype == ::apache::thrift::protocol::T_STRING) {
xfer += iprot->readString(this->workerid);
- isset_workerid = true;
+ this->__isset.workerid = true;
} else {
xfer += iprot->skip(ftype);
}
@@ -9111,7 +9107,7 @@ uint32_t ShowCompactResponseElement::rea
case 7:
if (ftype == ::apache::thrift::protocol::T_I64) {
xfer += iprot->readI64(this->start);
- isset_start = true;
+ this->__isset.start = true;
} else {
xfer += iprot->skip(ftype);
}
@@ -9119,7 +9115,7 @@ uint32_t ShowCompactResponseElement::rea
case 8:
if (ftype == ::apache::thrift::protocol::T_STRING) {
xfer += iprot->readString(this->runAs);
- isset_runAs = true;
+ this->__isset.runAs = true;
} else {
xfer += iprot->skip(ftype);
}
@@ -9137,18 +9133,10 @@ uint32_t ShowCompactResponseElement::rea
throw TProtocolException(TProtocolException::INVALID_DATA);
if (!isset_tablename)
throw TProtocolException(TProtocolException::INVALID_DATA);
- if (!isset_partitionname)
- throw TProtocolException(TProtocolException::INVALID_DATA);
if (!isset_type)
throw TProtocolException(TProtocolException::INVALID_DATA);
if (!isset_state)
throw TProtocolException(TProtocolException::INVALID_DATA);
- if (!isset_workerid)
- throw TProtocolException(TProtocolException::INVALID_DATA);
- if (!isset_start)
- throw TProtocolException(TProtocolException::INVALID_DATA);
- if (!isset_runAs)
- throw TProtocolException(TProtocolException::INVALID_DATA);
return xfer;
}
@@ -9164,10 +9152,11 @@ uint32_t ShowCompactResponseElement::wri
xfer += oprot->writeString(this->tablename);
xfer += oprot->writeFieldEnd();
- xfer += oprot->writeFieldBegin("partitionname", ::apache::thrift::protocol::T_STRING, 3);
- xfer += oprot->writeString(this->partitionname);
- xfer += oprot->writeFieldEnd();
-
+ if (this->__isset.partitionname) {
+ xfer += oprot->writeFieldBegin("partitionname", ::apache::thrift::protocol::T_STRING, 3);
+ xfer += oprot->writeString(this->partitionname);
+ xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldBegin("type", ::apache::thrift::protocol::T_I32, 4);
xfer += oprot->writeI32((int32_t)this->type);
xfer += oprot->writeFieldEnd();
@@ -9176,18 +9165,21 @@ uint32_t ShowCompactResponseElement::wri
xfer += oprot->writeString(this->state);
xfer += oprot->writeFieldEnd();
- xfer += oprot->writeFieldBegin("workerid", ::apache::thrift::protocol::T_STRING, 6);
- xfer += oprot->writeString(this->workerid);
- xfer += oprot->writeFieldEnd();
-
- xfer += oprot->writeFieldBegin("start", ::apache::thrift::protocol::T_I64, 7);
- xfer += oprot->writeI64(this->start);
- xfer += oprot->writeFieldEnd();
-
- xfer += oprot->writeFieldBegin("runAs", ::apache::thrift::protocol::T_STRING, 8);
- xfer += oprot->writeString(this->runAs);
- xfer += oprot->writeFieldEnd();
-
+ if (this->__isset.workerid) {
+ xfer += oprot->writeFieldBegin("workerid", ::apache::thrift::protocol::T_STRING, 6);
+ xfer += oprot->writeString(this->workerid);
+ xfer += oprot->writeFieldEnd();
+ }
+ if (this->__isset.start) {
+ xfer += oprot->writeFieldBegin("start", ::apache::thrift::protocol::T_I64, 7);
+ xfer += oprot->writeI64(this->start);
+ xfer += oprot->writeFieldEnd();
+ }
+ if (this->__isset.runAs) {
+ xfer += oprot->writeFieldBegin("runAs", ::apache::thrift::protocol::T_STRING, 8);
+ xfer += oprot->writeString(this->runAs);
+ xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -9203,10 +9195,11 @@ void swap(ShowCompactResponseElement &a,
swap(a.workerid, b.workerid);
swap(a.start, b.start);
swap(a.runAs, b.runAs);
+ swap(a.__isset, b.__isset);
}
-const char* ShowCompactResponse::ascii_fingerprint = "3CCBC4D398CA25527272FE78625DE88A";
-const uint8_t ShowCompactResponse::binary_fingerprint[16] = {0x3C,0xCB,0xC4,0xD3,0x98,0xCA,0x25,0x52,0x72,0x72,0xFE,0x78,0x62,0x5D,0xE8,0x8A};
+const char* ShowCompactResponse::ascii_fingerprint = "915B7B8DB8966D65769C0F98707BBAE3";
+const uint8_t ShowCompactResponse::binary_fingerprint[16] = {0x91,0x5B,0x7B,0x8D,0xB8,0x96,0x6D,0x65,0x76,0x9C,0x0F,0x98,0x70,0x7B,0xBA,0xE3};
uint32_t ShowCompactResponse::read(::apache::thrift::protocol::TProtocol* iprot) {
Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h Thu Sep 11 21:02:09 2014
@@ -4934,12 +4934,19 @@ class ShowCompactRequest {
void swap(ShowCompactRequest &a, ShowCompactRequest &b);
+typedef struct _ShowCompactResponseElement__isset {
+ _ShowCompactResponseElement__isset() : partitionname(false), workerid(false), start(false), runAs(false) {}
+ bool partitionname;
+ bool workerid;
+ bool start;
+ bool runAs;
+} _ShowCompactResponseElement__isset;
class ShowCompactResponseElement {
public:
- static const char* ascii_fingerprint; // = "42464F3A97707F984FDE462104223A69";
- static const uint8_t binary_fingerprint[16]; // = {0x42,0x46,0x4F,0x3A,0x97,0x70,0x7F,0x98,0x4F,0xDE,0x46,0x21,0x04,0x22,0x3A,0x69};
+ static const char* ascii_fingerprint; // = "2F338C265DC4FD82DD13F4966FE43F13";
+ static const uint8_t binary_fingerprint[16]; // = {0x2F,0x33,0x8C,0x26,0x5D,0xC4,0xFD,0x82,0xDD,0x13,0xF4,0x96,0x6F,0xE4,0x3F,0x13};
ShowCompactResponseElement() : dbname(), tablename(), partitionname(), type((CompactionType::type)0), state(), workerid(), start(0), runAs() {
}
@@ -4955,6 +4962,8 @@ class ShowCompactResponseElement {
int64_t start;
std::string runAs;
+ _ShowCompactResponseElement__isset __isset;
+
void __set_dbname(const std::string& val) {
dbname = val;
}
@@ -4965,6 +4974,7 @@ class ShowCompactResponseElement {
void __set_partitionname(const std::string& val) {
partitionname = val;
+ __isset.partitionname = true;
}
void __set_type(const CompactionType::type val) {
@@ -4977,14 +4987,17 @@ class ShowCompactResponseElement {
void __set_workerid(const std::string& val) {
workerid = val;
+ __isset.workerid = true;
}
void __set_start(const int64_t val) {
start = val;
+ __isset.start = true;
}
void __set_runAs(const std::string& val) {
runAs = val;
+ __isset.runAs = true;
}
bool operator == (const ShowCompactResponseElement & rhs) const
@@ -4993,17 +5006,25 @@ class ShowCompactResponseElement {
return false;
if (!(tablename == rhs.tablename))
return false;
- if (!(partitionname == rhs.partitionname))
+ if (__isset.partitionname != rhs.__isset.partitionname)
+ return false;
+ else if (__isset.partitionname && !(partitionname == rhs.partitionname))
return false;
if (!(type == rhs.type))
return false;
if (!(state == rhs.state))
return false;
- if (!(workerid == rhs.workerid))
+ if (__isset.workerid != rhs.__isset.workerid)
+ return false;
+ else if (__isset.workerid && !(workerid == rhs.workerid))
+ return false;
+ if (__isset.start != rhs.__isset.start)
+ return false;
+ else if (__isset.start && !(start == rhs.start))
return false;
- if (!(start == rhs.start))
+ if (__isset.runAs != rhs.__isset.runAs)
return false;
- if (!(runAs == rhs.runAs))
+ else if (__isset.runAs && !(runAs == rhs.runAs))
return false;
return true;
}
@@ -5024,8 +5045,8 @@ void swap(ShowCompactResponseElement &a,
class ShowCompactResponse {
public:
- static const char* ascii_fingerprint; // = "3CCBC4D398CA25527272FE78625DE88A";
- static const uint8_t binary_fingerprint[16]; // = {0x3C,0xCB,0xC4,0xD3,0x98,0xCA,0x25,0x52,0x72,0x72,0xFE,0x78,0x62,0x5D,0xE8,0x8A};
+ static const char* ascii_fingerprint; // = "915B7B8DB8966D65769C0F98707BBAE3";
+ static const uint8_t binary_fingerprint[16]; // = {0x91,0x5B,0x7B,0x8D,0xB8,0x96,0x6D,0x65,0x76,0x9C,0x0F,0x98,0x70,0x7B,0xBA,0xE3};
ShowCompactResponse() {
}
Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java Thu Sep 11 21:02:09 2014
@@ -135,9 +135,9 @@ public class FieldSchema implements org.
String comment)
{
this();
- this.name = org.apache.hive.common.util.HiveStringUtils.intern(name);
- this.type = org.apache.hive.common.util.HiveStringUtils.intern(type);
- this.comment = org.apache.hive.common.util.HiveStringUtils.intern(comment);
+ this.name = name;
+ this.type = type;
+ this.comment = comment;
}
/**
@@ -145,13 +145,13 @@ public class FieldSchema implements org.
*/
public FieldSchema(FieldSchema other) {
if (other.isSetName()) {
- this.name = org.apache.hive.common.util.HiveStringUtils.intern(other.name);
+ this.name = other.name;
}
if (other.isSetType()) {
- this.type = org.apache.hive.common.util.HiveStringUtils.intern(other.type);
+ this.type = other.type;
}
if (other.isSetComment()) {
- this.comment = org.apache.hive.common.util.HiveStringUtils.intern(other.comment);
+ this.comment = other.comment;
}
}
@@ -171,7 +171,7 @@ public class FieldSchema implements org.
}
public void setName(String name) {
- this.name = org.apache.hive.common.util.HiveStringUtils.intern(name);
+ this.name = name;
}
public void unsetName() {
@@ -194,7 +194,7 @@ public class FieldSchema implements org.
}
public void setType(String type) {
- this.type = org.apache.hive.common.util.HiveStringUtils.intern(type);
+ this.type = type;
}
public void unsetType() {
@@ -217,7 +217,7 @@ public class FieldSchema implements org.
}
public void setComment(String comment) {
- this.comment = org.apache.hive.common.util.HiveStringUtils.intern(comment);
+ this.comment = comment;
}
public void unsetComment() {
Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java Thu Sep 11 21:02:09 2014
@@ -182,14 +182,14 @@ public class Partition implements org.ap
{
this();
this.values = values;
- this.dbName = org.apache.hive.common.util.HiveStringUtils.intern(dbName);
- this.tableName = org.apache.hive.common.util.HiveStringUtils.intern(tableName);
+ this.dbName = dbName;
+ this.tableName = tableName;
this.createTime = createTime;
setCreateTimeIsSet(true);
this.lastAccessTime = lastAccessTime;
setLastAccessTimeIsSet(true);
this.sd = sd;
- this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
+ this.parameters = parameters;
}
/**
@@ -205,10 +205,10 @@ public class Partition implements org.ap
this.values = __this__values;
}
if (other.isSetDbName()) {
- this.dbName = org.apache.hive.common.util.HiveStringUtils.intern(other.dbName);
+ this.dbName = other.dbName;
}
if (other.isSetTableName()) {
- this.tableName = org.apache.hive.common.util.HiveStringUtils.intern(other.tableName);
+ this.tableName = other.tableName;
}
this.createTime = other.createTime;
this.lastAccessTime = other.lastAccessTime;
@@ -222,9 +222,9 @@ public class Partition implements org.ap
String other_element_key = other_element.getKey();
String other_element_value = other_element.getValue();
- String __this__parameters_copy_key = org.apache.hive.common.util.HiveStringUtils.intern(other_element_key);
+ String __this__parameters_copy_key = other_element_key;
- String __this__parameters_copy_value = org.apache.hive.common.util.HiveStringUtils.intern(other_element_value);
+ String __this__parameters_copy_value = other_element_value;
__this__parameters.put(__this__parameters_copy_key, __this__parameters_copy_value);
}
@@ -296,7 +296,7 @@ public class Partition implements org.ap
}
public void setDbName(String dbName) {
- this.dbName = org.apache.hive.common.util.HiveStringUtils.intern(dbName);
+ this.dbName = dbName;
}
public void unsetDbName() {
@@ -319,7 +319,7 @@ public class Partition implements org.ap
}
public void setTableName(String tableName) {
- this.tableName = org.apache.hive.common.util.HiveStringUtils.intern(tableName);
+ this.tableName = tableName;
}
public void unsetTableName() {
@@ -420,7 +420,7 @@ public class Partition implements org.ap
}
public void setParameters(Map<String,String> parameters) {
- this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
+ this.parameters = parameters;
}
public void unsetParameters() {
Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java Thu Sep 11 21:02:09 2014
@@ -137,9 +137,9 @@ public class SerDeInfo implements org.ap
Map<String,String> parameters)
{
this();
- this.name = org.apache.hive.common.util.HiveStringUtils.intern(name);
- this.serializationLib = org.apache.hive.common.util.HiveStringUtils.intern(serializationLib);
- this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
+ this.name = name;
+ this.serializationLib = serializationLib;
+ this.parameters = parameters;
}
/**
@@ -147,10 +147,10 @@ public class SerDeInfo implements org.ap
*/
public SerDeInfo(SerDeInfo other) {
if (other.isSetName()) {
- this.name = org.apache.hive.common.util.HiveStringUtils.intern(other.name);
+ this.name = other.name;
}
if (other.isSetSerializationLib()) {
- this.serializationLib = org.apache.hive.common.util.HiveStringUtils.intern(other.serializationLib);
+ this.serializationLib = other.serializationLib;
}
if (other.isSetParameters()) {
Map<String,String> __this__parameters = new HashMap<String,String>();
@@ -159,9 +159,9 @@ public class SerDeInfo implements org.ap
String other_element_key = other_element.getKey();
String other_element_value = other_element.getValue();
- String __this__parameters_copy_key = org.apache.hive.common.util.HiveStringUtils.intern(other_element_key);
+ String __this__parameters_copy_key = other_element_key;
- String __this__parameters_copy_value = org.apache.hive.common.util.HiveStringUtils.intern(other_element_value);
+ String __this__parameters_copy_value = other_element_value;
__this__parameters.put(__this__parameters_copy_key, __this__parameters_copy_value);
}
@@ -185,7 +185,7 @@ public class SerDeInfo implements org.ap
}
public void setName(String name) {
- this.name = org.apache.hive.common.util.HiveStringUtils.intern(name);
+ this.name = name;
}
public void unsetName() {
@@ -208,7 +208,7 @@ public class SerDeInfo implements org.ap
}
public void setSerializationLib(String serializationLib) {
- this.serializationLib = org.apache.hive.common.util.HiveStringUtils.intern(serializationLib);
+ this.serializationLib = serializationLib;
}
public void unsetSerializationLib() {
@@ -242,7 +242,7 @@ public class SerDeInfo implements org.ap
}
public void setParameters(Map<String,String> parameters) {
- this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
+ this.parameters = parameters;
}
public void unsetParameters() {
Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponseElement.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponseElement.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponseElement.java (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponseElement.java Thu Sep 11 21:02:09 2014
@@ -51,12 +51,12 @@ public class ShowCompactResponseElement
private String dbname; // required
private String tablename; // required
- private String partitionname; // required
+ private String partitionname; // optional
private CompactionType type; // required
private String state; // required
- private String workerid; // required
- private long start; // required
- private String runAs; // required
+ private String workerid; // optional
+ private long start; // optional
+ private String runAs; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -144,6 +144,7 @@ public class ShowCompactResponseElement
// isset id assignments
private static final int __START_ISSET_ID = 0;
private byte __isset_bitfield = 0;
+ private _Fields optionals[] = {_Fields.PARTITIONNAME,_Fields.WORKERID,_Fields.START,_Fields.RUN_AS};
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -151,17 +152,17 @@ public class ShowCompactResponseElement
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.TABLENAME, new org.apache.thrift.meta_data.FieldMetaData("tablename", org.apache.thrift.TFieldRequirementType.REQUIRED,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.PARTITIONNAME, new org.apache.thrift.meta_data.FieldMetaData("partitionname", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ tmpMap.put(_Fields.PARTITIONNAME, new org.apache.thrift.meta_data.FieldMetaData("partitionname", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.TYPE, new org.apache.thrift.meta_data.FieldMetaData("type", org.apache.thrift.TFieldRequirementType.REQUIRED,
new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, CompactionType.class)));
tmpMap.put(_Fields.STATE, new org.apache.thrift.meta_data.FieldMetaData("state", org.apache.thrift.TFieldRequirementType.REQUIRED,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.WORKERID, new org.apache.thrift.meta_data.FieldMetaData("workerid", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ tmpMap.put(_Fields.WORKERID, new org.apache.thrift.meta_data.FieldMetaData("workerid", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.START, new org.apache.thrift.meta_data.FieldMetaData("start", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ tmpMap.put(_Fields.START, new org.apache.thrift.meta_data.FieldMetaData("start", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- tmpMap.put(_Fields.RUN_AS, new org.apache.thrift.meta_data.FieldMetaData("runAs", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ tmpMap.put(_Fields.RUN_AS, new org.apache.thrift.meta_data.FieldMetaData("runAs", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ShowCompactResponseElement.class, metaDataMap);
@@ -173,23 +174,14 @@ public class ShowCompactResponseElement
public ShowCompactResponseElement(
String dbname,
String tablename,
- String partitionname,
CompactionType type,
- String state,
- String workerid,
- long start,
- String runAs)
+ String state)
{
this();
this.dbname = dbname;
this.tablename = tablename;
- this.partitionname = partitionname;
this.type = type;
this.state = state;
- this.workerid = workerid;
- this.start = start;
- setStartIsSet(true);
- this.runAs = runAs;
}
/**
@@ -622,8 +614,8 @@ public class ShowCompactResponseElement
return false;
}
- boolean this_present_start = true;
- boolean that_present_start = true;
+ boolean this_present_start = true && this.isSetStart();
+ boolean that_present_start = true && that.isSetStart();
if (this_present_start || that_present_start) {
if (!(this_present_start && that_present_start))
return false;
@@ -677,7 +669,7 @@ public class ShowCompactResponseElement
if (present_workerid)
builder.append(workerid);
- boolean present_start = true;
+ boolean present_start = true && (isSetStart());
builder.append(present_start);
if (present_start)
builder.append(start);
@@ -813,14 +805,16 @@ public class ShowCompactResponseElement
sb.append(this.tablename);
}
first = false;
- if (!first) sb.append(", ");
- sb.append("partitionname:");
- if (this.partitionname == null) {
- sb.append("null");
- } else {
- sb.append(this.partitionname);
+ if (isSetPartitionname()) {
+ if (!first) sb.append(", ");
+ sb.append("partitionname:");
+ if (this.partitionname == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.partitionname);
+ }
+ first = false;
}
- first = false;
if (!first) sb.append(", ");
sb.append("type:");
if (this.type == null) {
@@ -837,26 +831,32 @@ public class ShowCompactResponseElement
sb.append(this.state);
}
first = false;
- if (!first) sb.append(", ");
- sb.append("workerid:");
- if (this.workerid == null) {
- sb.append("null");
- } else {
- sb.append(this.workerid);
+ if (isSetWorkerid()) {
+ if (!first) sb.append(", ");
+ sb.append("workerid:");
+ if (this.workerid == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.workerid);
+ }
+ first = false;
}
- first = false;
- if (!first) sb.append(", ");
- sb.append("start:");
- sb.append(this.start);
- first = false;
- if (!first) sb.append(", ");
- sb.append("runAs:");
- if (this.runAs == null) {
- sb.append("null");
- } else {
- sb.append(this.runAs);
+ if (isSetStart()) {
+ if (!first) sb.append(", ");
+ sb.append("start:");
+ sb.append(this.start);
+ first = false;
+ }
+ if (isSetRunAs()) {
+ if (!first) sb.append(", ");
+ sb.append("runAs:");
+ if (this.runAs == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.runAs);
+ }
+ first = false;
}
- first = false;
sb.append(")");
return sb.toString();
}
@@ -871,10 +871,6 @@ public class ShowCompactResponseElement
throw new org.apache.thrift.protocol.TProtocolException("Required field 'tablename' is unset! Struct:" + toString());
}
- if (!isSetPartitionname()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'partitionname' is unset! Struct:" + toString());
- }
-
if (!isSetType()) {
throw new org.apache.thrift.protocol.TProtocolException("Required field 'type' is unset! Struct:" + toString());
}
@@ -883,18 +879,6 @@ public class ShowCompactResponseElement
throw new org.apache.thrift.protocol.TProtocolException("Required field 'state' is unset! Struct:" + toString());
}
- if (!isSetWorkerid()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'workerid' is unset! Struct:" + toString());
- }
-
- if (!isSetStart()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'start' is unset! Struct:" + toString());
- }
-
- if (!isSetRunAs()) {
- throw new org.apache.thrift.protocol.TProtocolException("Required field 'runAs' is unset! Struct:" + toString());
- }
-
// check for sub-struct validity
}
@@ -1022,9 +1006,11 @@ public class ShowCompactResponseElement
oprot.writeFieldEnd();
}
if (struct.partitionname != null) {
- oprot.writeFieldBegin(PARTITIONNAME_FIELD_DESC);
- oprot.writeString(struct.partitionname);
- oprot.writeFieldEnd();
+ if (struct.isSetPartitionname()) {
+ oprot.writeFieldBegin(PARTITIONNAME_FIELD_DESC);
+ oprot.writeString(struct.partitionname);
+ oprot.writeFieldEnd();
+ }
}
if (struct.type != null) {
oprot.writeFieldBegin(TYPE_FIELD_DESC);
@@ -1037,17 +1023,23 @@ public class ShowCompactResponseElement
oprot.writeFieldEnd();
}
if (struct.workerid != null) {
- oprot.writeFieldBegin(WORKERID_FIELD_DESC);
- oprot.writeString(struct.workerid);
+ if (struct.isSetWorkerid()) {
+ oprot.writeFieldBegin(WORKERID_FIELD_DESC);
+ oprot.writeString(struct.workerid);
+ oprot.writeFieldEnd();
+ }
+ }
+ if (struct.isSetStart()) {
+ oprot.writeFieldBegin(START_FIELD_DESC);
+ oprot.writeI64(struct.start);
oprot.writeFieldEnd();
}
- oprot.writeFieldBegin(START_FIELD_DESC);
- oprot.writeI64(struct.start);
- oprot.writeFieldEnd();
if (struct.runAs != null) {
- oprot.writeFieldBegin(RUN_AS_FIELD_DESC);
- oprot.writeString(struct.runAs);
- oprot.writeFieldEnd();
+ if (struct.isSetRunAs()) {
+ oprot.writeFieldBegin(RUN_AS_FIELD_DESC);
+ oprot.writeString(struct.runAs);
+ oprot.writeFieldEnd();
+ }
}
oprot.writeFieldStop();
oprot.writeStructEnd();
@@ -1068,12 +1060,34 @@ public class ShowCompactResponseElement
TTupleProtocol oprot = (TTupleProtocol) prot;
oprot.writeString(struct.dbname);
oprot.writeString(struct.tablename);
- oprot.writeString(struct.partitionname);
oprot.writeI32(struct.type.getValue());
oprot.writeString(struct.state);
- oprot.writeString(struct.workerid);
- oprot.writeI64(struct.start);
- oprot.writeString(struct.runAs);
+ BitSet optionals = new BitSet();
+ if (struct.isSetPartitionname()) {
+ optionals.set(0);
+ }
+ if (struct.isSetWorkerid()) {
+ optionals.set(1);
+ }
+ if (struct.isSetStart()) {
+ optionals.set(2);
+ }
+ if (struct.isSetRunAs()) {
+ optionals.set(3);
+ }
+ oprot.writeBitSet(optionals, 4);
+ if (struct.isSetPartitionname()) {
+ oprot.writeString(struct.partitionname);
+ }
+ if (struct.isSetWorkerid()) {
+ oprot.writeString(struct.workerid);
+ }
+ if (struct.isSetStart()) {
+ oprot.writeI64(struct.start);
+ }
+ if (struct.isSetRunAs()) {
+ oprot.writeString(struct.runAs);
+ }
}
@Override
@@ -1083,18 +1097,27 @@ public class ShowCompactResponseElement
struct.setDbnameIsSet(true);
struct.tablename = iprot.readString();
struct.setTablenameIsSet(true);
- struct.partitionname = iprot.readString();
- struct.setPartitionnameIsSet(true);
struct.type = CompactionType.findByValue(iprot.readI32());
struct.setTypeIsSet(true);
struct.state = iprot.readString();
struct.setStateIsSet(true);
- struct.workerid = iprot.readString();
- struct.setWorkeridIsSet(true);
- struct.start = iprot.readI64();
- struct.setStartIsSet(true);
- struct.runAs = iprot.readString();
- struct.setRunAsIsSet(true);
+ BitSet incoming = iprot.readBitSet(4);
+ if (incoming.get(0)) {
+ struct.partitionname = iprot.readString();
+ struct.setPartitionnameIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.workerid = iprot.readString();
+ struct.setWorkeridIsSet(true);
+ }
+ if (incoming.get(2)) {
+ struct.start = iprot.readI64();
+ struct.setStartIsSet(true);
+ }
+ if (incoming.get(3)) {
+ struct.runAs = iprot.readString();
+ struct.setRunAsIsSet(true);
+ }
}
}
Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java Thu Sep 11 21:02:09 2014
@@ -216,17 +216,17 @@ public class StorageDescriptor implement
{
this();
this.cols = cols;
- this.location = org.apache.hive.common.util.HiveStringUtils.intern(location);
- this.inputFormat = org.apache.hive.common.util.HiveStringUtils.intern(inputFormat);
- this.outputFormat = org.apache.hive.common.util.HiveStringUtils.intern(outputFormat);
+ this.location = location;
+ this.inputFormat = inputFormat;
+ this.outputFormat = outputFormat;
this.compressed = compressed;
setCompressedIsSet(true);
this.numBuckets = numBuckets;
setNumBucketsIsSet(true);
this.serdeInfo = serdeInfo;
- this.bucketCols = org.apache.hive.common.util.HiveStringUtils.intern(bucketCols);
+ this.bucketCols = bucketCols;
this.sortCols = sortCols;
- this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
+ this.parameters = parameters;
}
/**
@@ -242,13 +242,13 @@ public class StorageDescriptor implement
this.cols = __this__cols;
}
if (other.isSetLocation()) {
- this.location = org.apache.hive.common.util.HiveStringUtils.intern(other.location);
+ this.location = other.location;
}
if (other.isSetInputFormat()) {
- this.inputFormat = org.apache.hive.common.util.HiveStringUtils.intern(other.inputFormat);
+ this.inputFormat = other.inputFormat;
}
if (other.isSetOutputFormat()) {
- this.outputFormat = org.apache.hive.common.util.HiveStringUtils.intern(other.outputFormat);
+ this.outputFormat = other.outputFormat;
}
this.compressed = other.compressed;
this.numBuckets = other.numBuckets;
@@ -276,9 +276,9 @@ public class StorageDescriptor implement
String other_element_key = other_element.getKey();
String other_element_value = other_element.getValue();
- String __this__parameters_copy_key = org.apache.hive.common.util.HiveStringUtils.intern(other_element_key);
+ String __this__parameters_copy_key = other_element_key;
- String __this__parameters_copy_value = org.apache.hive.common.util.HiveStringUtils.intern(other_element_value);
+ String __this__parameters_copy_value = other_element_value;
__this__parameters.put(__this__parameters_copy_key, __this__parameters_copy_value);
}
@@ -356,7 +356,7 @@ public class StorageDescriptor implement
}
public void setLocation(String location) {
- this.location = org.apache.hive.common.util.HiveStringUtils.intern(location);
+ this.location = location;
}
public void unsetLocation() {
@@ -379,7 +379,7 @@ public class StorageDescriptor implement
}
public void setInputFormat(String inputFormat) {
- this.inputFormat = org.apache.hive.common.util.HiveStringUtils.intern(inputFormat);
+ this.inputFormat = inputFormat;
}
public void unsetInputFormat() {
@@ -402,7 +402,7 @@ public class StorageDescriptor implement
}
public void setOutputFormat(String outputFormat) {
- this.outputFormat = org.apache.hive.common.util.HiveStringUtils.intern(outputFormat);
+ this.outputFormat = outputFormat;
}
public void unsetOutputFormat() {
@@ -507,7 +507,7 @@ public class StorageDescriptor implement
}
public void setBucketCols(List<String> bucketCols) {
- this.bucketCols = org.apache.hive.common.util.HiveStringUtils.intern(bucketCols);
+ this.bucketCols = bucketCols;
}
public void unsetBucketCols() {
@@ -579,7 +579,7 @@ public class StorageDescriptor implement
}
public void setParameters(Map<String,String> parameters) {
- this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
+ this.parameters = parameters;
}
public void unsetParameters() {
Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py Thu Sep 11 21:02:09 2014
@@ -8054,18 +8054,10 @@ class ShowCompactResponseElement:
raise TProtocol.TProtocolException(message='Required field dbname is unset!')
if self.tablename is None:
raise TProtocol.TProtocolException(message='Required field tablename is unset!')
- if self.partitionname is None:
- raise TProtocol.TProtocolException(message='Required field partitionname is unset!')
if self.type is None:
raise TProtocol.TProtocolException(message='Required field type is unset!')
if self.state is None:
raise TProtocol.TProtocolException(message='Required field state is unset!')
- if self.workerid is None:
- raise TProtocol.TProtocolException(message='Required field workerid is unset!')
- if self.start is None:
- raise TProtocol.TProtocolException(message='Required field start is unset!')
- if self.runAs is None:
- raise TProtocol.TProtocolException(message='Required field runAs is unset!')
return
Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb Thu Sep 11 21:02:09 2014
@@ -1946,12 +1946,12 @@ class ShowCompactResponseElement
FIELDS = {
DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename'},
- PARTITIONNAME => {:type => ::Thrift::Types::STRING, :name => 'partitionname'},
+ PARTITIONNAME => {:type => ::Thrift::Types::STRING, :name => 'partitionname', :optional => true},
TYPE => {:type => ::Thrift::Types::I32, :name => 'type', :enum_class => ::CompactionType},
STATE => {:type => ::Thrift::Types::STRING, :name => 'state'},
- WORKERID => {:type => ::Thrift::Types::STRING, :name => 'workerid'},
- START => {:type => ::Thrift::Types::I64, :name => 'start'},
- RUNAS => {:type => ::Thrift::Types::STRING, :name => 'runAs'}
+ WORKERID => {:type => ::Thrift::Types::STRING, :name => 'workerid', :optional => true},
+ START => {:type => ::Thrift::Types::I64, :name => 'start', :optional => true},
+ RUNAS => {:type => ::Thrift::Types::STRING, :name => 'runAs', :optional => true}
}
def struct_fields; FIELDS; end
@@ -1959,12 +1959,8 @@ class ShowCompactResponseElement
def validate
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbname is unset!') unless @dbname
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tablename is unset!') unless @tablename
- raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partitionname is unset!') unless @partitionname
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field type is unset!') unless @type
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field state is unset!') unless @state
- raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field workerid is unset!') unless @workerid
- raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field start is unset!') unless @start
- raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field runAs is unset!') unless @runAs
unless @type.nil? || ::CompactionType::VALID_VALUES.include?(@type)
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field type!')
end
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Thu Sep 11 21:02:09 2014
@@ -44,6 +44,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.ql.exec.ConditionalTask;
import org.apache.hadoop.hive.ql.exec.FetchTask;
+import org.apache.hadoop.hive.ql.exec.MoveTask;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -1523,10 +1524,17 @@ public class Driver implements CommandPr
cxt.launching(tskRun);
// Launch Task
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.EXECPARALLEL) && tsk.isMapRedTask()) {
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.EXECPARALLEL)
+ && (tsk.isMapRedTask() || (tsk instanceof MoveTask))) {
// Launch it in the parallel mode, as a separate thread only for MR tasks
+ if (LOG.isInfoEnabled()){
+ LOG.info("Starting task [" + tsk + "] in parallel");
+ }
tskRun.start();
} else {
+ if (LOG.isInfoEnabled()){
+ LOG.info("Starting task [" + tsk + "] in serial mode");
+ }
tskRun.runSequential();
}
return tskRun;
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java Thu Sep 11 21:02:09 2014
@@ -19,8 +19,20 @@ package org.apache.hadoop.hive.ql.io.orc
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.EnumSet;
interface CompressionCodec {
+
+ public enum Modifier {
+ /* speed/compression tradeoffs */
+ FASTEST,
+ FAST,
+ DEFAULT,
+ /* data sensitivity modifiers */
+ TEXT,
+ BINARY
+ };
+
/**
* Compress the in buffer to the out buffer.
* @param in the bytes to compress
@@ -39,4 +51,17 @@ interface CompressionCodec {
* @throws IOException
*/
void decompress(ByteBuffer in, ByteBuffer out) throws IOException;
+
+ /**
+ * Produce a modified compression codec if the underlying algorithm allows
+ * modification.
+ *
+ * This does not modify the current object, but returns a new object if
+ * modifications are possible. Returns the same object if no modifications
+ * are possible.
+ * @param modifiers compression modifiers
+ * @return codec for use after optional modification
+ */
+ CompressionCodec modify(EnumSet<Modifier> modifiers);
+
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java Thu Sep 11 21:02:09 2014
@@ -101,6 +101,10 @@ public final class OrcFile {
SPEED, COMPRESSION;
}
+ public static enum CompressionStrategy {
+ SPEED, COMPRESSION;
+ }
+
// Note : these string definitions for table properties are deprecated,
// and retained only for backward compatibility, please do not add to
// them, add to OrcTableProperties below instead
@@ -230,6 +234,7 @@ public final class OrcFile {
private Version versionValue;
private WriterCallback callback;
private EncodingStrategy encodingStrategy;
+ private CompressionStrategy compressionStrategy;
private float paddingTolerance;
WriterOptions(Configuration conf) {
@@ -254,6 +259,15 @@ public final class OrcFile {
} else {
encodingStrategy = EncodingStrategy.valueOf(enString);
}
+
+ String compString = conf
+ .get(HiveConf.ConfVars.HIVE_ORC_COMPRESSION_STRATEGY.varname);
+ if (compString == null) {
+ compressionStrategy = CompressionStrategy.SPEED;
+ } else {
+ compressionStrategy = CompressionStrategy.valueOf(compString);
+ }
+
paddingTolerance =
conf.getFloat(HiveConf.ConfVars.HIVE_ORC_BLOCK_PADDING_TOLERANCE.varname,
HiveConf.ConfVars.HIVE_ORC_BLOCK_PADDING_TOLERANCE.defaultFloatVal);
@@ -403,7 +417,8 @@ public final class OrcFile {
opts.bufferSizeValue, opts.rowIndexStrideValue,
opts.memoryManagerValue, opts.blockPaddingValue,
opts.versionValue, opts.callback,
- opts.encodingStrategy, opts.paddingTolerance,
+ opts.encodingStrategy, opts.compressionStrategy,
+ opts.paddingTolerance,
opts.blockSizeValue);
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java Thu Sep 11 21:02:09 2014
@@ -25,6 +25,7 @@ import org.iq80.snappy.Snappy;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.EnumSet;
class SnappyCodec implements CompressionCodec, DirectDecompressionCodec {
@@ -99,4 +100,10 @@ class SnappyCodec implements Compression
decompressShim.decompress(in, out);
out.flip(); // flip for read
}
+
+ @Override
+ public CompressionCodec modify(EnumSet<Modifier> modifiers) {
+ // snappy allows no modifications
+ return this;
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Thu Sep 11 21:02:09 2014
@@ -24,6 +24,7 @@ import java.lang.management.ManagementFa
import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -37,6 +38,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.IOConstants;
+import org.apache.hadoop.hive.ql.io.orc.CompressionCodec.Modifier;
+import org.apache.hadoop.hive.ql.io.orc.OrcFile.CompressionStrategy;
import org.apache.hadoop.hive.ql.io.orc.OrcFile.EncodingStrategy;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics;
@@ -139,6 +142,7 @@ class WriterImpl implements Writer, Memo
private final OrcFile.WriterCallback callback;
private final OrcFile.WriterContext callbackContext;
private final OrcFile.EncodingStrategy encodingStrategy;
+ private final OrcFile.CompressionStrategy compressionStrategy;
WriterImpl(FileSystem fs,
Path path,
@@ -153,6 +157,7 @@ class WriterImpl implements Writer, Memo
OrcFile.Version version,
OrcFile.WriterCallback callback,
OrcFile.EncodingStrategy encodingStrategy,
+ CompressionStrategy compressionStrategy,
float paddingTolerance,
long blockSizeValue) throws IOException {
this.fs = fs;
@@ -174,6 +179,7 @@ class WriterImpl implements Writer, Memo
this.defaultStripeSize = stripeSize;
this.version = version;
this.encodingStrategy = encodingStrategy;
+ this.compressionStrategy = compressionStrategy;
this.addBlockPadding = addBlockPadding;
this.blockSize = blockSizeValue;
this.paddingTolerance = paddingTolerance;
@@ -447,10 +453,35 @@ class WriterImpl implements Writer, Memo
public OutStream createStream(int column,
OrcProto.Stream.Kind kind
) throws IOException {
- StreamName name = new StreamName(column, kind);
+ final StreamName name = new StreamName(column, kind);
+ final EnumSet<CompressionCodec.Modifier> modifiers;
+
+ switch (kind) {
+ case DATA:
+ case DICTIONARY_DATA:
+ if (getCompressionStrategy() == CompressionStrategy.SPEED) {
+ modifiers = EnumSet.of(Modifier.FAST, Modifier.TEXT);
+ } else {
+ modifiers = EnumSet.of(Modifier.DEFAULT, Modifier.TEXT);
+ }
+ break;
+ case LENGTH:
+ case DICTIONARY_COUNT:
+ case PRESENT:
+ case ROW_INDEX:
+ case SECONDARY:
+ // easily compressed using the fastest modes
+ modifiers = EnumSet.of(Modifier.FASTEST, Modifier.BINARY);
+ break;
+ default:
+ modifiers = null;
+ break;
+ }
+
BufferedStream result = streams.get(name);
if (result == null) {
- result = new BufferedStream(name.toString(), bufferSize, codec);
+ result = new BufferedStream(name.toString(), bufferSize,
+ codec == null ? codec : codec.modify(modifiers));
streams.put(name, result);
}
return result.outStream;
@@ -496,6 +527,14 @@ class WriterImpl implements Writer, Memo
}
/**
+ * Get the compression strategy to use.
+ * @return compression strategy
+ */
+ public CompressionStrategy getCompressionStrategy() {
+ return compressionStrategy;
+ }
+
+ /**
* Get the writer's configuration.
* @return configuration
*/
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java Thu Sep 11 21:02:09 2014
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io.orc
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.EnumSet;
import java.util.zip.DataFormatException;
import java.util.zip.Deflater;
import java.util.zip.Inflater;
@@ -32,10 +33,24 @@ class ZlibCodec implements CompressionCo
private Boolean direct = null;
+ private final int level;
+ private final int strategy;
+
+ public ZlibCodec() {
+ level = Deflater.DEFAULT_COMPRESSION;
+ strategy = Deflater.DEFAULT_STRATEGY;
+ }
+
+ private ZlibCodec(int level, int strategy) {
+ this.level = level;
+ this.strategy = strategy;
+ }
+
@Override
public boolean compress(ByteBuffer in, ByteBuffer out,
ByteBuffer overflow) throws IOException {
- Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true);
+ Deflater deflater = new Deflater(level, true);
+ deflater.setStrategy(strategy);
int length = in.remaining();
deflater.setInput(in.array(), in.arrayOffset() + in.position(), length);
deflater.finish();
@@ -113,4 +128,37 @@ class ZlibCodec implements CompressionCo
decompressShim.decompress(in, out);
out.flip(); // flip for read
}
+
+ @Override
+ public CompressionCodec modify(EnumSet<Modifier> modifiers) {
+ int l = this.level;
+ int s = this.strategy;
+
+ for (Modifier m : modifiers) {
+ switch (m) {
+ case BINARY:
+ /* filtered == less LZ77, more huffman */
+ s = Deflater.FILTERED;
+ break;
+ case TEXT:
+ s = Deflater.DEFAULT_STRATEGY;
+ break;
+ case FASTEST:
+ // deflate_fast looking for 8 byte patterns
+ l = Deflater.BEST_SPEED;
+ break;
+ case FAST:
+ // deflate_fast looking for 16 byte patterns
+ l = Deflater.BEST_SPEED + 1;
+ break;
+ case DEFAULT:
+ // deflate_slow looking for 128 byte patterns
+ l = Deflater.DEFAULT_COMPRESSION;
+ break;
+ default:
+ break;
+ }
+ }
+ return new ZlibCodec(l, s);
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/AppMasterEventProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/AppMasterEventProcessor.java?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/AppMasterEventProcessor.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/AppMasterEventProcessor.java Thu Sep 11 21:02:09 2014
@@ -56,6 +56,7 @@ public class AppMasterEventProcessor imp
} else {
events = new ArrayList<AppMasterEventOperator>();
}
+ events.add(event);
context.tsToEventMap.put(desc.getTableScan(), events);
return true;
}
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q Thu Sep 11 21:02:09 2014
@@ -39,3 +39,12 @@ and
d1.label in ('foo', 'bar')
GROUP BY d1.label
ORDER BY d1.label;
+
+EXPLAIN
+SELECT amount FROM agg_01, dim_shops WHERE dim_shops_id = id AND label = 'foo'
+UNION ALL
+SELECT amount FROM agg_01, dim_shops WHERE dim_shops_id = id AND label = 'bar';
+
+SELECT amount FROM agg_01, dim_shops WHERE dim_shops_id = id AND label = 'foo'
+UNION ALL
+SELECT amount FROM agg_01, dim_shops WHERE dim_shops_id = id AND label = 'bar';
\ No newline at end of file
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/groupby1.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/groupby1.q?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/groupby1.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/groupby1.q Thu Sep 11 21:02:09 2014
@@ -1,6 +1,8 @@
set hive.map.aggr=false;
set hive.groupby.skewindata=true;
+-- SORT_QUERY_RESULTS
+
CREATE TABLE dest_g1(key INT, value DOUBLE) STORED AS TEXTFILE;
set fs.default.name=invalidscheme:///;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/groupby4.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/groupby4.q?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/groupby4.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/groupby4.q Thu Sep 11 21:02:09 2014
@@ -1,6 +1,8 @@
set hive.map.aggr=false;
set hive.groupby.skewindata=true;
+-- SORT_QUERY_RESULTS
+
CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE;
EXPLAIN
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/having.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/having.q?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/having.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/having.q Thu Sep 11 21:02:09 2014
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
EXPLAIN SELECT count(value) AS c FROM src GROUP BY key HAVING c > 3;
SELECT count(value) AS c FROM src GROUP BY key HAVING c > 3;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/limit_pushdown.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/limit_pushdown.q?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/limit_pushdown.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/limit_pushdown.q Thu Sep 11 21:02:09 2014
@@ -1,6 +1,8 @@
set hive.limit.pushdown.memory.usage=0.3f;
set hive.optimize.reducededuplication.min.reducer=1;
+-- SORT_QUERY_RESULTS
+
-- HIVE-3562 Some limit can be pushed down to map stage
explain
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/merge1.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/merge1.q?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/merge1.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/merge1.q Thu Sep 11 21:02:09 2014
@@ -1,5 +1,7 @@
set hive.merge.mapredfiles=true;
+-- SORT_QUERY_RESULTS
+
create table dest1(key int, val int);
explain
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/merge2.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/merge2.q?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/merge2.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/merge2.q Thu Sep 11 21:02:09 2014
@@ -5,6 +5,8 @@ set mapred.min.split.size.per.node=256;
set mapred.min.split.size.per.rack=256;
set mapred.max.split.size=256;
+-- SORT_QUERY_RESULTS
+
create table test1(key int, val int);
explain
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_19.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_19.q?rev=1624389&r1=1624388&r2=1624389&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_19.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_19.q Thu Sep 11 21:02:09 2014
@@ -16,6 +16,8 @@ set mapred.input.dir.recursive=true;
-- Since this test creates sub-directories for the output table outputTbl1, it might be easier
-- to run the test only on hadoop 23
+-- SORT_QUERY_RESULTS
+
create table inputTbl1(key string, val string) stored as textfile;
create table outputTbl1(key string, values bigint) stored as textfile;