You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by to...@apache.org on 2012/03/09 21:35:20 UTC
svn commit: r1299060 - in /incubator/hcatalog/branches/branch-0.4: ./
storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/
Author: toffer
Date: Fri Mar 9 21:35:20 2012
New Revision: 1299060
URL: http://svn.apache.org/viewvc?rev=1299060&view=rev
Log:
HCATALOG-296 Hcatalog should be able talk to secure hbase server using hbase delegation tokens (rohini via toffer)
Added:
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseMapredUtil.java
Modified:
incubator/hcatalog/branches/branch-0.4/CHANGES.txt
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java
Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1299060&r1=1299059&r2=1299060&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Fri Mar 9 21:35:20 2012
@@ -40,6 +40,8 @@ Release 0.4.0 - Unreleased
HCAT-240. Changes to HCatOutputFormat to make it use SerDes instead of StorageDriver (toffer)
NEW FEATURES
+ HCAT-296 Hcatalog should be able talk to secure hbase server using hbase delegation tokens (rohini via toffer)
+
HCAT-249 Rework JSON StorageDriver into a JSON SerDe (khorgath via gates)
HCAT-255 Define hadoop properties on the hcat command line (ctdean via gates)
Modified: incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java?rev=1299060&r1=1299059&r2=1299060&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java Fri Mar 9 21:35:20 2012
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.FileOutputCommitter;
@@ -64,7 +65,8 @@ class HBaseBulkOutputFormat extends HBas
job.setOutputValueClass(Put.class);
job.setOutputCommitter(HBaseBulkOutputCommitter.class);
baseOutputFormat.checkOutputSpecs(ignored, job);
- getJTDelegationToken(job);
+ HBaseMapredUtil.addHBaseDelegationToken(job);
+ addJTDelegationToken(job);
}
@Override
@@ -76,10 +78,10 @@ class HBaseBulkOutputFormat extends HBas
ignored, job, name, progress), version);
}
- private void getJTDelegationToken(JobConf job) throws IOException {
+ private void addJTDelegationToken(JobConf job) throws IOException {
// Get jobTracker delegation token if security is enabled
// we need to launch the ImportSequenceFile job
- if (job.getBoolean("hadoop.security.authorization", false)) {
+ if (User.isSecurityEnabled()) {
JobClient jobClient = new JobClient(new JobConf(job));
try {
job.getCredentials().addToken(new Text("my mr token"),
Modified: incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java?rev=1299060&r1=1299059&r2=1299060&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java Fri Mar 9 21:35:20 2012
@@ -67,6 +67,7 @@ class HBaseDirectOutputFormat extends HB
job.setIfUnset(TableOutputFormat.OUTPUT_TABLE,
job.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY));
outputFormat.checkOutputSpecs(ignored, job);
+ HBaseMapredUtil.addHBaseDelegationToken(job);
}
private static class HBaseDirectRecordWriter implements
Modified: incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java?rev=1299060&r1=1299059&r2=1299060&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java Fri Mar 9 21:35:20 2012
@@ -105,6 +105,7 @@ class HBaseInputFormat implements InputF
public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf job, int numSplits)
throws IOException {
inputFormat.setConf(job);
+ HBaseMapredUtil.addHBaseDelegationToken(job);
return convertSplits(inputFormat.getSplits(HCatMapRedUtil.createJobContext(job, null,
Reporter.NULL)));
}
Added: incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseMapredUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseMapredUtil.java?rev=1299060&view=auto
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseMapredUtil.java (added)
+++ incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseMapredUtil.java Fri Mar 9 21:35:20 2012
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.mapred.JobConf;
+
+public class HBaseMapredUtil {
+
+ private HBaseMapredUtil() {
+ }
+
+ /**
+ * Get delegation token from hbase and add it to JobConf
+ * @param job
+ * @throws IOException
+ */
+ public static void addHBaseDelegationToken(JobConf job) throws IOException {
+ if (User.isHBaseSecurityEnabled(job)) {
+ try {
+ User.getCurrent().obtainAuthTokenForJob(job);
+ } catch (InterruptedException e) {
+ throw new IOException("Error while obtaining hbase delegation token", e);
+ }
+ }
+ }
+
+}
Modified: incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java?rev=1299060&r1=1299059&r2=1299060&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java Fri Mar 9 21:35:20 2012
@@ -220,6 +220,7 @@ class ImportSequenceFile {
}
conf.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName);
+ conf.setBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, false);
boolean localMode = "local".equals(conf.get("mapred.job.tracker"));