You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/11/14 00:03:03 UTC
svn commit: r1541755 [2/2] - in /hive/branches/tez: ./ cli/ hbase-handler/
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/queries/negative/ hcatalog/ hcatalog/build-suppo...
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java Wed Nov 13 23:03:01 2013
@@ -21,12 +21,33 @@ package org.apache.hcatalog.hbase.snapsh
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
+import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.AbortWriteTransactionRequest;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.AbortWriteTransactionResponse;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.BeginWriteTransactionRequest;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.BeginWriteTransactionResponse;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.CommitWriteTransactionRequest;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.CommitWriteTransactionResponse;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.CreateSnapshotRequest;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.CreateSnapshotResponse;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.CreateTableRequest;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.CreateTableResponse;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.DropTableRequest;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.DropTableResponse;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.GetAbortedWriteTransactionsRequest;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.GetAbortedWriteTransactionsResponse;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.KeepAliveTransactionRequest;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.KeepAliveTransactionResponse;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpointProtos.RevisionManagerEndpointService;
/**
* This class is nothing but a delegate for the enclosed proxy,
@@ -34,17 +55,18 @@ import org.apache.hadoop.hbase.util.Byte
*/
public class RevisionManagerEndpointClient implements RevisionManager, Configurable {
- private Configuration conf = null;
- private RevisionManager rmProxy;
+ private final RPCConverter rpcConverter = new RPCConverter();
+ private Configuration conf;
+ private HTable htable;
@Override
public Configuration getConf() {
- return this.conf;
+ return conf;
}
@Override
- public void setConf(Configuration arg0) {
- this.conf = arg0;
+ public void setConf(Configuration conf) {
+ this.conf = conf;
}
@Override
@@ -56,70 +78,207 @@ public class RevisionManagerEndpointClie
public void open() throws IOException {
// clone to adjust RPC settings unique to proxy
Configuration clonedConf = new Configuration(conf);
- // conf.set("hbase.ipc.client.connect.max.retries", "0");
- // conf.setInt(HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS, 1);
clonedConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); // do not retry RPC
- HTable table = new HTable(clonedConf, HConstants.ROOT_TABLE_NAME);
- rmProxy = table.coprocessorProxy(RevisionManagerProtocol.class,
- Bytes.toBytes("anyRow"));
- rmProxy.open();
+ htable = new HTable(clonedConf, TableName.META_TABLE_NAME.getNameAsString());
}
@Override
public void close() throws IOException {
- rmProxy.close();
+ htable.close();
}
@Override
- public void createTable(String table, List<String> columnFamilies) throws IOException {
- rmProxy.createTable(table, columnFamilies);
+ public void createTable(final String table, final List<String> columnFamilies) throws IOException {
+ call(new Batch.Call<RevisionManagerEndpointService, Void>() {
+ @Override
+ public Void call(RevisionManagerEndpointService service)
+ throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<CreateTableResponse> done =
+ new BlockingRpcCallback<CreateTableResponse>();
+ CreateTableRequest request = CreateTableRequest.newBuilder()
+ .setTableName(table).addAllColumnFamilies(columnFamilies).build();
+ service.createTable(controller, request, done);
+ blockOnResponse(done, controller);
+ return null;
+ }
+ });
}
@Override
- public void dropTable(String table) throws IOException {
- rmProxy.dropTable(table);
+ public void dropTable(final String table) throws IOException {
+ call(new Batch.Call<RevisionManagerEndpointService, Void>() {
+ @Override
+ public Void call(RevisionManagerEndpointService service)
+ throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<DropTableResponse> done =
+ new BlockingRpcCallback<DropTableResponse>();
+ DropTableRequest request = DropTableRequest.newBuilder()
+ .setTableName(table).build();
+ service.dropTable(null, request, done);
+ blockOnResponse(done, controller);
+ return null;
+ }
+ });
}
@Override
- public Transaction beginWriteTransaction(String table, List<String> families) throws IOException {
- return rmProxy.beginWriteTransaction(table, families);
+ public Transaction beginWriteTransaction(final String table, final List<String> families) throws IOException {
+ return beginWriteTransaction(table, families, null);
}
@Override
- public Transaction beginWriteTransaction(String table, List<String> families, long keepAlive)
+ public Transaction beginWriteTransaction(final String table, final List<String> families, final Long keepAlive)
throws IOException {
- return rmProxy.beginWriteTransaction(table, families, keepAlive);
+ return call(new Batch.Call<RevisionManagerEndpointService, Transaction>() {
+ @Override
+ public Transaction call(RevisionManagerEndpointService service)
+ throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<BeginWriteTransactionResponse> done =
+ new BlockingRpcCallback<BeginWriteTransactionResponse>();
+ BeginWriteTransactionRequest.Builder builder = BeginWriteTransactionRequest.newBuilder()
+ .setTableName(table)
+ .addAllColumnFamilies(families);
+ if(keepAlive != null) {
+ builder.setKeepAlive(keepAlive);
+ }
+ service.beginWriteTransaction(controller, builder.build(), done);
+ return rpcConverter.convertTransaction(blockOnResponse(done, controller).getTransaction());
+ }
+ });
}
@Override
- public void commitWriteTransaction(Transaction transaction) throws IOException {
- rmProxy.commitWriteTransaction(transaction);
+ public void commitWriteTransaction(final Transaction transaction) throws IOException {
+ call(new Batch.Call<RevisionManagerEndpointService, Void>() {
+ @Override
+ public Void call(RevisionManagerEndpointService service)
+ throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<CommitWriteTransactionResponse> done =
+ new BlockingRpcCallback<CommitWriteTransactionResponse>();
+ CommitWriteTransactionRequest request = CommitWriteTransactionRequest.newBuilder()
+ .setTransaction(rpcConverter.convertTransaction(transaction)).build();
+ service.commitWriteTransaction(controller, request, done);
+ blockOnResponse(done, controller);
+ return null;
+ }
+ });
}
@Override
- public void abortWriteTransaction(Transaction transaction) throws IOException {
- rmProxy.abortWriteTransaction(transaction);
+ public void abortWriteTransaction(final Transaction transaction) throws IOException {
+ call(new Batch.Call<RevisionManagerEndpointService, Void>() {
+ @Override
+ public Void call(RevisionManagerEndpointService service)
+ throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<AbortWriteTransactionResponse> done =
+ new BlockingRpcCallback<AbortWriteTransactionResponse>();
+ AbortWriteTransactionRequest request = AbortWriteTransactionRequest.newBuilder()
+ .setTransaction(rpcConverter.convertTransaction(transaction)).build();
+ service.abortWriteTransaction(controller, request, done);
+ blockOnResponse(done, controller);
+ return null;
+ }
+ });
}
@Override
- public List<FamilyRevision> getAbortedWriteTransactions(String table, String columnFamily)
+ public List<FamilyRevision> getAbortedWriteTransactions(final String table, final String columnFamily)
throws IOException {
- return rmProxy.getAbortedWriteTransactions(table, columnFamily);
+ return call(new Batch.Call<RevisionManagerEndpointService, List<FamilyRevision>>() {
+ @Override
+ public List<FamilyRevision> call(RevisionManagerEndpointService service)
+ throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<GetAbortedWriteTransactionsResponse> done =
+ new BlockingRpcCallback<GetAbortedWriteTransactionsResponse>();
+ GetAbortedWriteTransactionsRequest request = GetAbortedWriteTransactionsRequest.newBuilder()
+ .setTableName(table)
+ .setColumnFamily(columnFamily)
+ .build();
+ service.getAbortedWriteTransactions(controller, request, done);
+ return rpcConverter.convertFamilyRevisions(blockOnResponse(done, controller).getFamilyRevisionsList());
+ }
+ });
}
@Override
public TableSnapshot createSnapshot(String tableName) throws IOException {
- return rmProxy.createSnapshot(tableName);
+ return createSnapshot(tableName, null);
}
@Override
- public TableSnapshot createSnapshot(String tableName, long revision) throws IOException {
- return rmProxy.createSnapshot(tableName, revision);
+ public TableSnapshot createSnapshot(final String tableName, final Long revision) throws IOException {
+ return call(new Batch.Call<RevisionManagerEndpointService, TableSnapshot>() {
+ @Override
+ public TableSnapshot call(RevisionManagerEndpointService service)
+ throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<CreateSnapshotResponse> done =
+ new BlockingRpcCallback<CreateSnapshotResponse>();
+ CreateSnapshotRequest.Builder builder = CreateSnapshotRequest.newBuilder()
+ .setTableName(tableName);
+ if(revision != null) {
+ builder.setRevision(revision);
+ }
+ service.createSnapshot(controller, builder.build(), done);
+ return rpcConverter.convertTableSnapshot(blockOnResponse(done, controller).getTableSnapshot());
+ }
+ });
}
@Override
- public void keepAlive(Transaction transaction) throws IOException {
- rmProxy.keepAlive(transaction);
+ public void keepAlive(final Transaction transaction) throws IOException {
+ call(new Batch.Call<RevisionManagerEndpointService, Void>() {
+ @Override
+ public Void call(RevisionManagerEndpointService service)
+ throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<KeepAliveTransactionResponse> done =
+ new BlockingRpcCallback<KeepAliveTransactionResponse>();
+ KeepAliveTransactionRequest request = KeepAliveTransactionRequest.newBuilder()
+ .setTransaction(rpcConverter.convertTransaction(transaction)).build();
+ service.keepAliveTransaction(controller, request, done);
+ blockOnResponse(done, controller);
+ return null;
+ }
+ });
+ }
+ private <R> R blockOnResponse(BlockingRpcCallback<R> done, ServerRpcController controller)
+ throws IOException {
+ R response = done.get();
+ if(controller.failedOnException()) {
+ throw controller.getFailedOn();
+ }
+ if(controller.failed()) {
+ String error = controller.errorText();
+ if(error == null) {
+ error = "Server indicated failure but error text was empty";
+ }
+ throw new RuntimeException(error);
+ }
+ return response;
+ }
+ private <R> R call(Batch.Call<RevisionManagerEndpointService, R> callable) throws IOException {
+ try {
+ Map<byte[], R> result = htable.coprocessorService(RevisionManagerEndpointService.class, null, null, callable);
+ if(result.isEmpty()) {
+ return null;
+ }
+ return result.values().iterator().next();
+ } catch(IOException e) {
+ throw (IOException)e;
+ } catch(RuntimeException e) {
+ throw (RuntimeException)e;
+ } catch(Error e) {
+ throw (Error)e;
+ } catch(Throwable throwable) {
+ throw new RuntimeException(throwable);
+ }
}
}
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java Wed Nov 13 23:03:01 2013
@@ -59,7 +59,14 @@ public class TableSnapshot implements Se
* @return List<String> A list of column families associated with the snapshot.
*/
public List<String> getColumnFamilies(){
- return new ArrayList<String>(this.cfRevisionMap.keySet());
+ return new ArrayList<String>(this.cfRevisionMap.keySet());
+ }
+
+ /**
+ * For wire serialization only
+ */
+ Map<String, Long> getColumnFamilyRevisionMap() {
+ return cfRevisionMap;
}
/**
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java Wed Nov 13 23:03:01 2013
@@ -64,6 +64,13 @@ public class Transaction implements Seri
}
/**
+ * For wire serialization only
+ */
+ long getTimeStamp() {
+ return timeStamp;
+ }
+
+ /**
* @return The expire timestamp associated with a transaction.
*/
long getTransactionExpireTimeStamp() {
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java Wed Nov 13 23:03:01 2013
@@ -119,7 +119,7 @@ public class ZKBasedRevisionManager impl
* @see org.apache.hcatalog.hbase.snapshot.RevisionManager#beginWriteTransaction(java.lang.String, java.util.List, long)
*/
public Transaction beginWriteTransaction(String table,
- List<String> families, long keepAlive) throws IOException {
+ List<String> families, Long keepAlive) throws IOException {
checkInputParams(table, families);
zkUtil.setUpZnodesForTable(table, families);
@@ -175,7 +175,7 @@ public class ZKBasedRevisionManager impl
*/
public Transaction beginWriteTransaction(String table, List<String> families)
throws IOException {
- return beginWriteTransaction(table, families, -1);
+ return beginWriteTransaction(table, families, -1L);
}
/**
@@ -352,7 +352,7 @@ public class ZKBasedRevisionManager impl
/* @throws IOException
* @see org.apache.hcatalog.hbase.snapshot.RevsionManager#createSnapshot(java.lang.String, long)
*/
- public TableSnapshot createSnapshot(String tableName, long revision) throws IOException {
+ public TableSnapshot createSnapshot(String tableName, Long revision) throws IOException {
long currentID = zkUtil.currentID(tableName);
if (revision > currentID) {
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java Wed Nov 13 23:03:01 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
@@ -121,7 +122,7 @@ public class ManyMiniCluster {
protected synchronized void stop() {
if (hbaseCluster != null) {
- HConnectionManager.deleteAllConnections(true);
+ HConnectionManager.deleteAllConnections();
try {
hbaseCluster.shutdown();
} catch (Exception e) {
@@ -266,7 +267,7 @@ public class ManyMiniCluster {
hbaseCluster = new MiniHBaseCluster(hbaseConf, numRegionServers);
hbaseConf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort());
//opening the META table ensures that cluster is running
- new HTable(hbaseConf, HConstants.META_TABLE_NAME);
+ new HTable(hbaseConf, TableName.META_TABLE_NAME.getNameAsString());
} catch (Exception e) {
throw new IllegalStateException("Failed to setup HBase Cluster", e);
}
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java Wed Nov 13 23:03:01 2013
@@ -19,6 +19,15 @@
package org.apache.hcatalog.hbase;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -71,15 +80,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
/**
* Tests components of HBaseHCatStorageHandler using ManyMiniCluster.
* Including ImprtSequenceFile and HBaseBulkOutputFormat
@@ -258,6 +258,7 @@ public class TestHBaseBulkOutputFormat e
}
index++;
}
+ table.close();
//test if load count is the same
assertEquals(data.length, index);
//test if scratch directory was erased
@@ -297,6 +298,7 @@ public class TestHBaseBulkOutputFormat e
//create job
+ HBaseHCatStorageHandler.setHBaseSerializers(conf);
Job job = new Job(conf, testName);
job.setWorkingDirectory(new Path(methodTestDir, "mr_work"));
job.setJarByClass(this.getClass());
@@ -335,6 +337,7 @@ public class TestHBaseBulkOutputFormat e
}
index++;
}
+ table.close();
//test if load count is the same
assertEquals(data.length, index);
//test if scratch directory was erased
@@ -432,6 +435,7 @@ public class TestHBaseBulkOutputFormat e
}
index++;
}
+ table.close();
//test if load count is the same
assertEquals(data.length, index);
}
@@ -514,6 +518,7 @@ public class TestHBaseBulkOutputFormat e
}
index++;
}
+ table.close();
//test if load count is the same
assertEquals(data.length, index);
}
@@ -608,6 +613,7 @@ public class TestHBaseBulkOutputFormat e
job.setOutputKeyClass(BytesWritable.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(0);
+ table.close();
assertTrue(job.waitForCompletion(true));
}
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java Wed Nov 13 23:03:01 2013
@@ -189,6 +189,7 @@ public class TestHBaseDirectOutputFormat
}
index++;
}
+ table.close();
assertEquals(data.length, index);
}
@@ -356,6 +357,7 @@ public class TestHBaseDirectOutputFormat
}
count++;
}
+ table.close();
assertEquals(data.length - 1, count);
// verify that the inputformat returns empty results.
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java Wed Nov 13 23:03:01 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.util.Byte
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.hbase.ResultWritable;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -546,7 +547,7 @@ public class TestHCatHBaseInputFormat ex
}
static class MapReadProjectionHTable
- implements org.apache.hadoop.mapred.Mapper<ImmutableBytesWritable, Result, WritableComparable<?>, Text> {
+ implements org.apache.hadoop.mapred.Mapper<ImmutableBytesWritable, Object, WritableComparable<?>, Text> {
static boolean error = false;
static int count = 0;
@@ -560,9 +561,17 @@ public class TestHCatHBaseInputFormat ex
}
@Override
- public void map(ImmutableBytesWritable key, Result result,
+ public void map(ImmutableBytesWritable key, Object resultObj,
OutputCollector<WritableComparable<?>, Text> output, Reporter reporter)
throws IOException {
+ Result result;
+ if (resultObj instanceof Result){
+ result = (Result) resultObj;
+ } else if (resultObj instanceof ResultWritable) {
+ result = ((ResultWritable)resultObj).getResult();
+ } else {
+ throw new IllegalArgumentException("Illegal Argument " + (resultObj == null ? "null" : resultObj.getClass().getName()));
+ }
System.out.println("Result " + result.toString());
List<KeyValue> list = result.list();
boolean correctValues = (list.size() == 1)
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java Wed Nov 13 23:03:01 2013
@@ -210,7 +210,7 @@ public class TestRevisionManager extends
String tableName = newTableName("testTable");
List<String> columnFamilies = Arrays.asList("cf1", "cf2");
Transaction txn = manager.beginWriteTransaction(tableName,
- columnFamilies, 40);
+ columnFamilies, 40L);
Thread.sleep(100);
try {
manager.commitWriteTransaction(txn);
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java Wed Nov 13 23:03:01 2013
@@ -20,6 +20,7 @@ package org.apache.hcatalog.hbase.snapsh
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -135,13 +136,13 @@ public class TestRevisionManagerEndpoint
@Override
public Transaction beginWriteTransaction(String table,
List<String> families) throws IOException {
- return recordCall(null, table, families);
+ return recordCall(new Transaction(table, families, 0L, 0L), table, families);
}
@Override
public Transaction beginWriteTransaction(String table,
- List<String> families, long keepAlive) throws IOException {
- return recordCall(null, table, families, keepAlive);
+ List<String> families, Long keepAlive) throws IOException {
+ return recordCall(new Transaction(table, families, 0L, 0L), table, families, keepAlive);
}
@Override
@@ -157,17 +158,17 @@ public class TestRevisionManagerEndpoint
@Override
public List<FamilyRevision> getAbortedWriteTransactions(String table,
String columnFamily) throws IOException {
- return null;
+ return Collections.singletonList(new FamilyRevision(0L, 0L));
}
@Override
public TableSnapshot createSnapshot(String tableName)
throws IOException {
- return null;
+ return createSnapshot(tableName, 0L);
}
@Override
- public TableSnapshot createSnapshot(String tableName, long revision)
+ public TableSnapshot createSnapshot(String tableName, Long revision)
throws IOException {
TableSnapshot ret = new TableSnapshot(tableName, new HashMap<String, Long>(), revision);
return recordCall(ret, tableName, revision);
@@ -201,7 +202,7 @@ public class TestRevisionManagerEndpoint
Assert.assertEquals(call.methodName, call, mockImpl.lastCall);
call = new MockRM.Invocation("createSnapshot", null, "t3", 1L);
- call.ret = rm.createSnapshot("t3", 1);
+ call.ret = rm.createSnapshot("t3", 1L);
Assert.assertEquals(call.methodName, call, mockImpl.lastCall);
}
Modified: hive/branches/tez/itests/hcatalog-unit/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hcatalog-unit/pom.xml?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/itests/hcatalog-unit/pom.xml (original)
+++ hive/branches/tez/itests/hcatalog-unit/pom.xml Wed Nov 13 23:03:01 2013
@@ -97,18 +97,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <version>${hbase.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
@@ -143,6 +131,64 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop1-compat</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop1-compat</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
@@ -199,6 +245,64 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
Modified: hive/branches/tez/itests/qtest/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/qtest/pom.xml?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/itests/qtest/pom.xml (original)
+++ hive/branches/tez/itests/qtest/pom.xml Wed Nov 13 23:03:01 2013
@@ -106,19 +106,6 @@
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <version>${hbase.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <version>${hbase.version}</version>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
</dependencies>
<profiles>
<profile>
@@ -156,6 +143,58 @@
<version>${hadoop-20S.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop1-compat</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop1-compat</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</profile>
<profile>
@@ -196,6 +235,58 @@
<artifactId>commons-logging</artifactId>
<version>${commons-logging.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</profile>
</profiles>
Modified: hive/branches/tez/itests/util/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/util/pom.xml?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/itests/util/pom.xml (original)
+++ hive/branches/tez/itests/util/pom.xml Wed Nov 13 23:03:01 2013
@@ -65,17 +65,6 @@
<artifactId>junit</artifactId>
<version>${junit.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <version>${hbase.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <version>${hbase.version}</version>
- <classifier>tests</classifier>
- </dependency>
</dependencies>
<profiles>
@@ -95,6 +84,27 @@
<artifactId>hadoop-test</artifactId>
<version>${hadoop-20S.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop1.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
</dependencies>
</profile>
<profile>
@@ -122,6 +132,27 @@
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop-23.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
</dependencies>
</profile>
</profiles>
Modified: hive/branches/tez/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/pom.xml?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/pom.xml (original)
+++ hive/branches/tez/pom.xml Wed Nov 13 23:03:01 2013
@@ -104,7 +104,8 @@
<hadoop-20.version>0.20.2</hadoop-20.version>
<hadoop-20S.version>1.2.1</hadoop-20S.version>
<hadoop-23.version>2.2.0</hadoop-23.version>
- <hbase.version>0.94.6.1</hbase.version>
+ <hbase.hadoop1.version>0.96.0-hadoop1</hbase.hadoop1.version>
+ <hbase.hadoop2.version>0.96.0-hadoop2</hbase.hadoop2.version>
<httpcomponents.version>4.1.3</httpcomponents.version>
<jackson.version>1.9.2</jackson.version>
<javaewah.version>0.3.2</javaewah.version>
Modified: hive/branches/tez/ql/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/pom.xml?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/ql/pom.xml (original)
+++ hive/branches/tez/ql/pom.xml Wed Nov 13 23:03:01 2013
@@ -237,7 +237,7 @@
</dependency>
</dependencies>
</profile>
- <profile>
+ <profile>
<id>hadoop-2</id>
<dependencies>
<dependency>
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java Wed Nov 13 23:03:01 2013
@@ -43,12 +43,16 @@ import org.apache.hadoop.hive.ql.exec.Ut
import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.InputFormatChecker;
+import org.apache.hadoop.hive.ql.io.orc.Metadata;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.FileGenerator;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.SplitGenerator;
import org.apache.hadoop.hive.ql.io.orc.Reader.FileMetaInfo;
+import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -179,7 +183,8 @@ public class OrcInputFormat implements
public static SearchArgument createSarg(List<OrcProto.Type> types, Configuration conf) {
String serializedPushdown = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
if (serializedPushdown == null
- || conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR) == null) {
+ || (conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR) == null
+ && conf.get(serdeConstants.LIST_COLUMNS) == null)) {
LOG.info("No ORC pushdown predicate");
return null;
}
@@ -324,6 +329,7 @@ public class OrcInputFormat implements
* the different worker threads.
*/
static class Context {
+ private final Configuration conf;
private static Cache<Path, FileInfo> footerCache;
private final ExecutorService threadPool;
private final List<OrcSplit> splits = new ArrayList<OrcSplit>(10000);
@@ -344,6 +350,7 @@ public class OrcInputFormat implements
private int schedulers = 0;
Context(Configuration conf) {
+ this.conf = conf;
minSize = conf.getLong(MIN_SPLIT_SIZE, DEFAULT_MIN_SPLIT_SIZE);
maxSize = conf.getLong(MAX_SPLIT_SIZE, DEFAULT_MAX_SPLIT_SIZE);
footerInSplits = HiveConf.getBoolVar(conf, ConfVars.HIVE_ORC_INCLUDE_FILE_FOOTER_IN_SPLITS);
@@ -528,6 +535,8 @@ public class OrcInputFormat implements
private final FileInfo fileInfo;
private Iterable<StripeInformation> stripes;
private FileMetaInfo fileMetaInfo;
+ private Metadata metadata;
+ private List<OrcProto.Type> types;
SplitGenerator(Context context, FileSystem fs,
@@ -635,9 +644,49 @@ public class OrcInputFormat implements
public void run() {
try {
populateAndCacheStripeDetails();
+ Configuration conf = context.conf;
+ SearchArgument sarg = createSarg(types, conf);
+ List<StripeStatistics> stripeStats = null;
+ int[] filterColumns = null;
+ if (sarg != null) {
+ List<PredicateLeaf> sargLeaves = null;
+ String[] columnNames = conf.get(serdeConstants.LIST_COLUMNS).split(",");
+ if (columnNames == null) {
+ columnNames = conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR).split(",");
+ }
+ sargLeaves = sarg.getLeaves();
+ filterColumns = new int[sargLeaves.size()];
+ for (int i = 0; i < filterColumns.length; ++i) {
+ String colName = sargLeaves.get(i).getColumnName();
+ filterColumns[i] = RecordReaderImpl.findColumns(columnNames, colName);
+ }
+
+ stripeStats = metadata.getStripeStatistics();
+ }
+
long currentOffset = -1;
long currentLength = 0;
+ int idx = -1;
for(StripeInformation stripe: stripes) {
+ idx++;
+
+ // eliminate stripes that doesn't satisfy the predicate condition
+ if (sarg != null && !isStripeSatisfyPredicate(stripeStats.get(idx), sarg, filterColumns)) {
+
+ // if a stripe doesn't satisfy predicate condition then skip it
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Eliminating ORC stripe-" + idx + " of file '" + file.getPath()
+ + "' as it did not satisfy predicate condition.");
+ }
+
+ // create split for the previous unfinished stripe
+ if (currentOffset != -1) {
+ createSplit(currentOffset, currentLength, fileMetaInfo);
+ currentOffset = -1;
+ }
+ continue;
+ }
+
// if we are working on a stripe, over the min stripe size, and
// crossed a block boundary, cut the input split here.
if (currentOffset != -1 && currentLength > context.minSize &&
@@ -675,9 +724,6 @@ public class OrcInputFormat implements
}
}
-
-
-
private void populateAndCacheStripeDetails() {
try {
Reader orcReader;
@@ -686,20 +732,27 @@ public class OrcInputFormat implements
found = true;
stripes = fileInfo.stripeInfos;
fileMetaInfo = fileInfo.fileMetaInfo;
+ metadata = fileInfo.metadata;
+ types = fileInfo.types;
// For multiple runs, in case sendSplitsInFooter changes
if (fileMetaInfo == null && context.footerInSplits) {
orcReader = OrcFile.createReader(fs, file.getPath());
fileInfo.fileMetaInfo = orcReader.getFileMetaInfo();
+ fileInfo.metadata = orcReader.getMetadata();
+ fileInfo.types = orcReader.getTypes();
}
}
if (!found) {
orcReader = OrcFile.createReader(fs, file.getPath());
stripes = orcReader.getStripes();
+ metadata = orcReader.getMetadata();
+ types = orcReader.getTypes();
fileMetaInfo = context.footerInSplits ? orcReader.getFileMetaInfo() : null;
if (context.cacheStripeDetails) {
// Populate into cache.
Context.footerCache.put(file.getPath(),
- new FileInfo(file.getModificationTime(), file.getLen(), stripes, fileMetaInfo));
+ new FileInfo(file.getModificationTime(), file.getLen(), stripes, metadata,
+ types, fileMetaInfo));
}
}
} catch (Throwable th) {
@@ -715,6 +768,54 @@ public class OrcInputFormat implements
}
}
+ private boolean isStripeSatisfyPredicate(StripeStatistics stripeStatistics,
+ SearchArgument sarg, int[] filterColumns) {
+ if (sarg != null && filterColumns != null) {
+ List<PredicateLeaf> predLeaves = sarg.getLeaves();
+ TruthValue[] truthValues = new TruthValue[predLeaves.size()];
+ for (int pred = 0; pred < truthValues.length; pred++) {
+ if (filterColumns[pred] != -1) {
+
+ // column statistics at index 0 contains only the number of rows
+ ColumnStatistics stats = stripeStatistics.getColumnStatistics()[filterColumns[pred] + 1];
+ Object minValue = getMin(stats);
+ Object maxValue = getMax(stats);
+ truthValues[pred] = RecordReaderImpl.evaluatePredicateRange(predLeaves.get(pred),
+ minValue, maxValue);
+ }
+ }
+ return sarg.evaluate(truthValues).isNeeded();
+ }
+ return true;
+ }
+
+ private Object getMax(ColumnStatistics index) {
+ if (index instanceof IntegerColumnStatistics) {
+ return ((IntegerColumnStatistics) index).getMaximum();
+ } else if (index instanceof DoubleColumnStatistics) {
+ return ((DoubleColumnStatistics) index).getMaximum();
+ } else if (index instanceof StringColumnStatistics) {
+ return ((StringColumnStatistics) index).getMaximum();
+ } else if (index instanceof DateColumnStatistics) {
+ return ((DateColumnStatistics) index).getMaximum();
+ } else {
+ return null;
+ }
+ }
+
+ private Object getMin(ColumnStatistics index) {
+ if (index instanceof IntegerColumnStatistics) {
+ return ((IntegerColumnStatistics) index).getMinimum();
+ } else if (index instanceof DoubleColumnStatistics) {
+ return ((DoubleColumnStatistics) index).getMinimum();
+ } else if (index instanceof StringColumnStatistics) {
+ return ((StringColumnStatistics) index).getMinimum();
+ } else if (index instanceof DateColumnStatistics) {
+ return ((DateColumnStatistics) index).getMinimum();
+ } else {
+ return null;
+ }
+ }
}
@Override
@@ -762,13 +863,18 @@ public class OrcInputFormat implements
long size;
Iterable<StripeInformation> stripeInfos;
FileMetaInfo fileMetaInfo;
+ Metadata metadata;
+ List<OrcProto.Type> types;
+
- FileInfo(long modificationTime, long size, Iterable<StripeInformation> stripeInfos,
- FileMetaInfo fileMetaInfo) {
+ FileInfo(long modificationTime, long size, Iterable<StripeInformation> stripeInfos,
+ Metadata metadata, List<OrcProto.Type> types, FileMetaInfo fileMetaInfo) {
this.modificationTime = modificationTime;
this.size = size;
this.stripeInfos = stripeInfos;
this.fileMetaInfo = fileMetaInfo;
+ this.metadata = metadata;
+ this.types = types;
}
}
}
\ No newline at end of file
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Wed Nov 13 23:03:01 2013
@@ -133,7 +133,7 @@ class RecordReaderImpl implements Record
advanceToNextRow(0L);
}
- private static int findColumns(String[] columnNames,
+ static int findColumns(String[] columnNames,
String columnName) {
for(int i=0; i < columnNames.length; ++i) {
if (columnName.equals(columnNames[i])) {
@@ -2034,6 +2034,11 @@ class RecordReaderImpl implements Record
}
}
Object maxValue = getMax(index);
+ return evaluatePredicateRange(predicate, minValue, maxValue);
+ }
+
+ static TruthValue evaluatePredicateRange(PredicateLeaf predicate, Object minValue,
+ Object maxValue) {
Location loc;
switch (predicate.getOperator()) {
case NULL_SAFE_EQUALS:
@@ -2154,7 +2159,7 @@ class RecordReaderImpl implements Record
leafValues[pred] = TruthValue.YES_NO_NULL;
}
}
- result[rowGroup] = sarg.evaluate(leafValues).isNotNeeded();
+ result[rowGroup] = sarg.evaluate(leafValues).isNeeded();
if (LOG.isDebugEnabled()) {
LOG.debug("Row group " + (rowIndexStride * rowGroup) + " to " +
(rowIndexStride * (rowGroup+1) - 1) + " is " +
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java Wed Nov 13 23:03:01 2013
@@ -139,7 +139,7 @@ public interface SearchArgument {
* Does the RecordReader need to include this set of records?
* @return true unless none of the rows qualify
*/
- public boolean isNotNeeded() {
+ public boolean isNeeded() {
switch (this) {
case NO:
case NULL:
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java Wed Nov 13 23:03:01 2013
@@ -201,7 +201,9 @@ public class ParseDriver {
throw new ParseException(parser.errors);
}
- return (ASTNode) r.getTree();
+ ASTNode tree = (ASTNode) r.getTree();
+ tree.setUnknownTokenBoundaries();
+ return tree;
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Nov 13 23:03:01 2013
@@ -1824,6 +1824,69 @@ public class SemanticAnalyzer extends Ba
}
}
+ private void extractJoinCondsFromWhereClause(QBJoinTree joinTree, QB qb, String dest, ASTNode predicate) throws SemanticException {
+
+ switch (predicate.getType()) {
+ case HiveParser.KW_AND:
+ extractJoinCondsFromWhereClause(joinTree, qb, dest, (ASTNode) predicate.getChild(0));
+ extractJoinCondsFromWhereClause(joinTree, qb, dest, (ASTNode) predicate.getChild(1));
+ break;
+ case HiveParser.EQUAL_NS:
+ case HiveParser.EQUAL:
+
+ ASTNode leftCondn = (ASTNode) predicate.getChild(0);
+ ArrayList<String> leftCondAl1 = new ArrayList<String>();
+ ArrayList<String> leftCondAl2 = new ArrayList<String>();
+ try {
+ parseJoinCondPopulateAlias(joinTree, leftCondn, leftCondAl1, leftCondAl2,
+ null);
+ } catch(SemanticException se) {
+ // suppress here; if it is a real issue will get caught in where clause handling.
+ return;
+ }
+
+ ASTNode rightCondn = (ASTNode) predicate.getChild(1);
+ ArrayList<String> rightCondAl1 = new ArrayList<String>();
+ ArrayList<String> rightCondAl2 = new ArrayList<String>();
+ try {
+ parseJoinCondPopulateAlias(joinTree, rightCondn, rightCondAl1,
+ rightCondAl2, null);
+ } catch(SemanticException se) {
+ // suppress here; if it is a real issue will get caught in where clause handling.
+ return;
+ }
+
+ if (((leftCondAl1.size() != 0) && (leftCondAl2.size() != 0))
+ || ((rightCondAl1.size() != 0) && (rightCondAl2.size() != 0))) {
+ // this is not a join condition.
+ return;
+ }
+
+ if (((leftCondAl1.size() == 0) && (leftCondAl2.size() == 0))
+ || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) {
+ // this is not a join condition. Will get handled by predicate pushdown.
+ return;
+ }
+
+ List<String> leftSrc = new ArrayList<String>();
+ JoinCond cond = joinTree.getJoinCond()[0];
+ JoinType type = cond.getJoinType();
+ applyEqualityPredicateToQBJoinTree(joinTree, type, leftSrc,
+ predicate, leftCondn, rightCondn,
+ leftCondAl1, leftCondAl2,
+ rightCondAl1, rightCondAl2);
+ if (leftSrc.size() == 1) {
+ joinTree.setLeftAlias(leftSrc.get(0));
+ }
+
+ // todo: hold onto this predicate, so that we don't add it to the Filter Operator.
+
+ break;
+ default:
+ return;
+ }
+ }
+
@SuppressWarnings("nls")
public <T extends OperatorDesc> Operator<T> putOpInsertMap(Operator<T> op,
RowResolver rr) {
@@ -8508,6 +8571,18 @@ public class SemanticAnalyzer extends Ba
} else {
QBJoinTree joinTree = genJoinTree(qb, joinExpr, aliasToOpInfo);
qb.setQbJoinTree(joinTree);
+ /*
+ * if there is only one destintaion in Query try to push where predicates
+ * as Join conditions
+ */
+ Set<String> dests = qb.getParseInfo().getClauseNames();
+ if ( dests.size() == 1 ) {
+ String dest = dests.iterator().next();
+ ASTNode whereClause = qb.getParseInfo().getWhrForClause(dest);
+ if ( whereClause != null ) {
+ extractJoinCondsFromWhereClause(joinTree, qb, dest, (ASTNode) whereClause.getChild(0) );
+ }
+ }
mergeJoinTree(qb);
}
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/create_view_translate.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/create_view_translate.q?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/create_view_translate.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/create_view_translate.q Wed Nov 13 23:03:01 2013
@@ -11,3 +11,14 @@ describe formatted w;
drop view v;
drop view w;
+
+
+-- HIVE-4116 Can't use views using map datatype.
+
+CREATE TABLE items (id INT, name STRING, info MAP<STRING,STRING>);
+
+explain
+CREATE VIEW priceview AS SELECT items.id, items.info['price'] FROM items;
+CREATE VIEW priceview AS SELECT items.id, items.info['price'] FROM items;
+
+select * from priceview;
Modified: hive/branches/tez/ql/src/test/results/clientpositive/create_view_translate.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/create_view_translate.q.out?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/create_view_translate.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/create_view_translate.q.out Wed Nov 13 23:03:01 2013
@@ -99,3 +99,52 @@ POSTHOOK: query: drop view w
POSTHOOK: type: DROPVIEW
POSTHOOK: Input: default@w
POSTHOOK: Output: default@w
+PREHOOK: query: -- HIVE-4116 Can't use views using map datatype.
+
+CREATE TABLE items (id INT, name STRING, info MAP<STRING,STRING>)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- HIVE-4116 Can't use views using map datatype.
+
+CREATE TABLE items (id INT, name STRING, info MAP<STRING,STRING>)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@items
+PREHOOK: query: explain
+CREATE VIEW priceview AS SELECT items.id, items.info['price'] FROM items
+PREHOOK: type: CREATEVIEW
+POSTHOOK: query: explain
+CREATE VIEW priceview AS SELECT items.id, items.info['price'] FROM items
+POSTHOOK: type: CREATEVIEW
+ABSTRACT SYNTAX TREE:
+ (TOK_CREATEVIEW (TOK_TABNAME priceview) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME items))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL items) id)) (TOK_SELEXPR ([ (. (TOK_TABLE_OR_COL items) info) 'price'))))))
+
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Create View Operator:
+ Create View
+ if not exists: false
+ is alter view as select: false
+ or replace: false
+ columns: id int, _c1 string
+ expanded text: SELECT `items`.`id`, `items`.`info`['price'] FROM `default`.`items`
+ name: priceview
+ original text: SELECT items.id, items.info['price'] FROM items
+
+
+PREHOOK: query: CREATE VIEW priceview AS SELECT items.id, items.info['price'] FROM items
+PREHOOK: type: CREATEVIEW
+POSTHOOK: query: CREATE VIEW priceview AS SELECT items.id, items.info['price'] FROM items
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Output: default@priceview
+PREHOOK: query: select * from priceview
+PREHOOK: type: QUERY
+PREHOOK: Input: default@items
+PREHOOK: Input: default@priceview
+#### A masked pattern was here ####
+POSTHOOK: query: select * from priceview
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@items
+POSTHOOK: Input: default@priceview
+#### A masked pattern was here ####
Modified: hive/branches/tez/ql/src/test/results/clientpositive/sample8.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/sample8.q.out?rev=1541755&r1=1541754&r2=1541755&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/sample8.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/sample8.q.out Wed Nov 13 23:03:01 2013
@@ -35,10 +35,20 @@ STAGE PLANS:
Filter Operator
isSamplingPred: true
predicate:
- expr: (((hash(key) & 2147483647) % 1) = 0)
+ expr: ((((hash(key) & 2147483647) % 10) = 0) and (((hash(key) & 2147483647) % 1) = 0))
type: boolean
Reduce Output Operator
- sort order:
+ key expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ expr: value
+ type: string
tag: 0
value expressions:
expr: key
@@ -56,10 +66,20 @@ STAGE PLANS:
Filter Operator
isSamplingPred: true
predicate:
- expr: (((hash(key) & 2147483647) % 10) = 0)
+ expr: ((((hash(key) & 2147483647) % 1) = 0) and (((hash(key) & 2147483647) % 10) = 0))
type: boolean
Reduce Output Operator
- sort order:
+ key expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ expr: value
+ type: string
tag: 1
value expressions:
expr: key