You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/08 20:43:38 UTC
[48/51] [partial] hive git commit: Revert "HIVE-14671 : merge master
into hive-14535 (Wei Zheng)"
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/predicate/TestAccumuloRangeGenerator.java
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/predicate/TestAccumuloRangeGenerator.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/predicate/TestAccumuloRangeGenerator.java
index 5f3baab..339da07 100644
--- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/predicate/TestAccumuloRangeGenerator.java
+++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/predicate/TestAccumuloRangeGenerator.java
@@ -16,15 +16,20 @@
*/
package org.apache.hadoop.hive.accumulo.predicate;
-import com.google.common.collect.Lists;
+import static org.junit.Assert.assertNotNull;
+
+import java.sql.Date;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.accumulo.AccumuloHiveConstants;
-import org.apache.hadoop.hive.accumulo.TestAccumuloDefaultIndexScanner;
import org.apache.hadoop.hive.accumulo.columns.ColumnEncoding;
import org.apache.hadoop.hive.accumulo.columns.HiveAccumuloRowIdColumnMapping;
-import org.apache.hadoop.hive.accumulo.serde.AccumuloSerDeParameters;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -37,29 +42,22 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.udf.UDFLike;
import org.apache.hadoop.hive.ql.udf.UDFToString;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPlus;
-import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.sql.Date;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-
-import static org.junit.Assert.assertNotNull;
+import com.google.common.collect.Lists;
/**
*
@@ -68,14 +66,12 @@ public class TestAccumuloRangeGenerator {
private AccumuloPredicateHandler handler;
private HiveAccumuloRowIdColumnMapping rowIdMapping;
- private Configuration conf;
@Before
public void setup() {
handler = AccumuloPredicateHandler.getInstance();
rowIdMapping = new HiveAccumuloRowIdColumnMapping(AccumuloHiveConstants.ROWID,
- ColumnEncoding.STRING,"row", TypeInfoFactory.stringTypeInfo.toString());
- conf = new Configuration(true);
+ ColumnEncoding.STRING, "row", TypeInfoFactory.stringTypeInfo.toString());
}
@Test
@@ -112,7 +108,7 @@ public class TestAccumuloRangeGenerator {
List<Range> expectedRanges = Arrays
.asList(new Range(new Key("f"), true, new Key("m\0"), false));
- AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(conf, handler, rowIdMapping, "rid");
+ AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(handler, rowIdMapping, "rid");
Dispatcher disp = new DefaultRuleDispatcher(rangeGenerator,
Collections.<Rule,NodeProcessor> emptyMap(), null);
GraphWalker ogw = new DefaultGraphWalker(disp);
@@ -167,7 +163,7 @@ public class TestAccumuloRangeGenerator {
// Should generate (-inf,+inf)
List<Range> expectedRanges = Arrays.asList(new Range());
- AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(conf, handler, rowIdMapping, "rid");
+ AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(handler, rowIdMapping, "rid");
Dispatcher disp = new DefaultRuleDispatcher(rangeGenerator,
Collections.<Rule,NodeProcessor> emptyMap(), null);
GraphWalker ogw = new DefaultGraphWalker(disp);
@@ -240,7 +236,7 @@ public class TestAccumuloRangeGenerator {
// Should generate ['q', +inf)
List<Range> expectedRanges = Arrays.asList(new Range(new Key("q"), true, null, false));
- AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(conf, handler, rowIdMapping, "rid");
+ AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(handler, rowIdMapping, "rid");
Dispatcher disp = new DefaultRuleDispatcher(rangeGenerator,
Collections.<Rule,NodeProcessor> emptyMap(), null);
GraphWalker ogw = new DefaultGraphWalker(disp);
@@ -295,7 +291,7 @@ public class TestAccumuloRangeGenerator {
// Should generate [f,+inf)
List<Range> expectedRanges = Arrays.asList(new Range(new Key("f"), true, null, false));
- AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(conf, handler, rowIdMapping, "rid");
+ AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(handler, rowIdMapping, "rid");
Dispatcher disp = new DefaultRuleDispatcher(rangeGenerator,
Collections.<Rule,NodeProcessor> emptyMap(), null);
GraphWalker ogw = new DefaultGraphWalker(disp);
@@ -353,7 +349,7 @@ public class TestAccumuloRangeGenerator {
List<Range> expectedRanges = Arrays.asList(new Range(new Key("2014-01-01"), true, new Key(
"2014-07-01"), false));
- AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(conf, handler, rowIdMapping, "rid");
+ AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(handler, rowIdMapping, "rid");
Dispatcher disp = new DefaultRuleDispatcher(rangeGenerator,
Collections.<Rule,NodeProcessor> emptyMap(), null);
GraphWalker ogw = new DefaultGraphWalker(disp);
@@ -401,7 +397,7 @@ public class TestAccumuloRangeGenerator {
ExprNodeGenericFuncDesc node = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,
new GenericUDFOPEqualOrGreaterThan(), Arrays.asList(key, cast));
- AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(conf, handler, rowIdMapping, "key");
+ AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(handler, rowIdMapping, "key");
Dispatcher disp = new DefaultRuleDispatcher(rangeGenerator,
Collections.<Rule,NodeProcessor> emptyMap(), null);
GraphWalker ogw = new DefaultGraphWalker(disp);
@@ -450,7 +446,7 @@ public class TestAccumuloRangeGenerator {
ExprNodeGenericFuncDesc both = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,
new GenericUDFOPAnd(), bothFilters);
- AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(conf, handler, rowIdMapping, "rid");
+ AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(handler, rowIdMapping, "rid");
Dispatcher disp = new DefaultRuleDispatcher(rangeGenerator,
Collections.<Rule,NodeProcessor> emptyMap(), null);
GraphWalker ogw = new DefaultGraphWalker(disp);
@@ -468,161 +464,4 @@ public class TestAccumuloRangeGenerator {
Object result = nodeOutput.get(both);
Assert.assertNull(result);
}
-
- @Test
- public void testRangeOverStringIndexedField() throws Exception {
- // age >= '10'
- ExprNodeDesc column = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "age", null, false);
- ExprNodeDesc constant = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, "10");
- List<ExprNodeDesc> children = Lists.newArrayList();
- children.add(column);
- children.add(constant);
- ExprNodeDesc node = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,
- new GenericUDFOPEqualOrGreaterThan(), children);
- assertNotNull(node);
-
- // age <= '50'
- ExprNodeDesc column2 = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "age", null,
- false);
- ExprNodeDesc constant2 = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, "50");
- List<ExprNodeDesc> children2 = Lists.newArrayList();
- children2.add(column2);
- children2.add(constant2);
- ExprNodeDesc node2 = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,
- new GenericUDFOPEqualOrLessThan(), children2);
- assertNotNull(node2);
-
- // And UDF
- List<ExprNodeDesc> bothFilters = Lists.newArrayList();
- bothFilters.add(node);
- bothFilters.add(node2);
- ExprNodeGenericFuncDesc both = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,
- new GenericUDFOPAnd(), bothFilters);
-
- AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(conf, handler, rowIdMapping, "rid");
- rangeGenerator.setIndexScanner(TestAccumuloDefaultIndexScanner.buildMockHandler(10));
- Dispatcher disp = new DefaultRuleDispatcher(rangeGenerator,
- Collections.<Rule,NodeProcessor> emptyMap(), null);
- GraphWalker ogw = new DefaultGraphWalker(disp);
- ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.add(both);
- HashMap<Node,Object> nodeOutput = new HashMap<Node,Object>();
-
- try {
- ogw.startWalking(topNodes, nodeOutput);
- } catch (SemanticException ex) {
- throw new RuntimeException(ex);
- }
-
- // Filters are using an index which should match 3 rows
- Object result = nodeOutput.get(both);
- if ( result instanceof List) {
- List results = (List) result;
- Assert.assertEquals(3, results.size());
- Assert.assertTrue("does not contain row1", results.contains(new Range("row1")));
- Assert.assertTrue("does not contain row2", results.contains(new Range("row2")));
- Assert.assertTrue("does not contain row3", results.contains(new Range("row3")));
- } else {
- Assert.fail("Results not a list");
- }
- }
-
- @Test
- public void testRangeOverIntegerIndexedField() throws Exception {
- // cars >= 2
- ExprNodeDesc column = new ExprNodeColumnDesc(TypeInfoFactory.intTypeInfo, "cars", null, false);
- ExprNodeDesc constant = new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo, 2);
- List<ExprNodeDesc> children = Lists.newArrayList();
- children.add(column);
- children.add(constant);
- ExprNodeDesc node = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
- new GenericUDFOPEqualOrGreaterThan(), children);
- assertNotNull(node);
-
- // cars <= 9
- ExprNodeDesc column2 = new ExprNodeColumnDesc(TypeInfoFactory.intTypeInfo, "cars", null,
- false);
- ExprNodeDesc constant2 = new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo, 9);
- List<ExprNodeDesc> children2 = Lists.newArrayList();
- children2.add(column2);
- children2.add(constant2);
- ExprNodeDesc node2 = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
- new GenericUDFOPEqualOrLessThan(), children2);
- assertNotNull(node2);
-
- // And UDF
- List<ExprNodeDesc> bothFilters = Lists.newArrayList();
- bothFilters.add(node);
- bothFilters.add(node2);
- ExprNodeGenericFuncDesc both = new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,
- new GenericUDFOPAnd(), bothFilters);
-
- AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(conf, handler, rowIdMapping, "rid");
- rangeGenerator.setIndexScanner(TestAccumuloDefaultIndexScanner.buildMockHandler(10));
- Dispatcher disp = new DefaultRuleDispatcher(rangeGenerator,
- Collections.<Rule,NodeProcessor> emptyMap(), null);
- GraphWalker ogw = new DefaultGraphWalker(disp);
- ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.add(both);
- HashMap<Node,Object> nodeOutput = new HashMap<Node,Object>();
-
- try {
- ogw.startWalking(topNodes, nodeOutput);
- } catch (SemanticException ex) {
- throw new RuntimeException(ex);
- }
-
- // Filters are using an index which should match 3 rows
- Object result = nodeOutput.get(both);
- if ( result instanceof List) {
- List results = (List) result;
- Assert.assertEquals(3, results.size());
- Assert.assertTrue("does not contain row1", results.contains(new Range("row1")));
- Assert.assertTrue("does not contain row2", results.contains(new Range("row2")));
- Assert.assertTrue("does not contain row3", results.contains(new Range("row3")));
- } else {
- Assert.fail("Results not a list");
- }
- }
-
- @Test
- public void testRangeOverBooleanIndexedField() throws Exception {
- // mgr == true
- ExprNodeDesc column = new ExprNodeColumnDesc(TypeInfoFactory.booleanTypeInfo, "mgr", null, false);
- ExprNodeDesc constant = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, true);
- List<ExprNodeDesc> children = Lists.newArrayList();
- children.add(column);
- children.add(constant);
- ExprNodeDesc node = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
- new GenericUDFOPEqual(), children);
- assertNotNull(node);
-
- AccumuloRangeGenerator rangeGenerator = new AccumuloRangeGenerator(conf, handler, rowIdMapping, "rid");
- rangeGenerator.setIndexScanner(TestAccumuloDefaultIndexScanner.buildMockHandler(10));
- Dispatcher disp = new DefaultRuleDispatcher(rangeGenerator,
- Collections.<Rule,NodeProcessor> emptyMap(), null);
- GraphWalker ogw = new DefaultGraphWalker(disp);
- ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.add(node);
- HashMap<Node,Object> nodeOutput = new HashMap<Node,Object>();
-
- try {
- ogw.startWalking(topNodes, nodeOutput);
- } catch (SemanticException ex) {
- throw new RuntimeException(ex);
- }
-
- // Filters are using an index which should match 2 rows
- Object result = nodeOutput.get(node);
- if ( result instanceof List) {
- List results = (List) result;
- Assert.assertEquals(2, results.size());
- Assert.assertTrue("does not contain row1", results.contains( new Range( "row1")));
- Assert.assertTrue("does not contain row3", results.contains( new Range( "row3")));
- }
- else {
- Assert.fail("Results not a list");
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/accumulo-handler/src/test/queries/positive/accumulo_index.q
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_index.q b/accumulo-handler/src/test/queries/positive/accumulo_index.q
deleted file mode 100644
index 52a33af..0000000
--- a/accumulo-handler/src/test/queries/positive/accumulo_index.q
+++ /dev/null
@@ -1,44 +0,0 @@
-DROP TABLE accumulo_index_test;
-
-CREATE TABLE accumulo_index_test (
- rowid string,
- active boolean,
- num_offices tinyint,
- num_personel smallint,
- total_manhours int,
- num_shareholders bigint,
- eff_rating float,
- err_rating double,
- yearly_production decimal,
- start_date date,
- address varchar(100),
- phone char(13),
- last_update timestamp )
-ROW FORMAT SERDE 'org.apache.hadoop.hive.accumulo.serde.AccumuloSerDe'
-STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
-WITH SERDEPROPERTIES (
- "accumulo.columns.mapping" = ":rowID,a:act,a:off,a:per,a:mhs,a:shs,a:eff,a:err,a:yp,a:sd,a:addr,a:ph,a:lu",
- "accumulo.table.name"="accumulo_index_test",
- "accumulo.indexed.columns"="*",
- "accumulo.indextable.name"="accumulo_index_idx"
- );
-
-
-insert into accumulo_index_test values( "row1", true, 55, 107, 555555, 1223232332,
- 4.5, 0.8, 1232223, "2001-10-10", "123 main street",
- "555-555-5555", "2016-02-22 12:45:07.000000000");
-
-select * from accumulo_index_test where active = 'true';
-select * from accumulo_index_test where num_offices = 55;
-select * from accumulo_index_test where num_personel = 107;
-select * from accumulo_index_test where total_manhours < 555556;
-select * from accumulo_index_test where num_shareholders >= 1223232331;
-select * from accumulo_index_test where eff_rating <= 4.5;
-select * from accumulo_index_test where err_rating >= 0.8;
-select * from accumulo_index_test where yearly_production = 1232223;
-select * from accumulo_index_test where start_date = "2001-10-10";
-select * from accumulo_index_test where address >= "100 main street";
-select * from accumulo_index_test where phone <= "555-555-5555";
-select * from accumulo_index_test where last_update >= "2016-02-22 12:45:07";
-
-DROP TABLE accumulo_index_test;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/accumulo-handler/src/test/results/positive/accumulo_index.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_index.q.out b/accumulo-handler/src/test/results/positive/accumulo_index.q.out
deleted file mode 100644
index 5cb3d73..0000000
--- a/accumulo-handler/src/test/results/positive/accumulo_index.q.out
+++ /dev/null
@@ -1,180 +0,0 @@
-PREHOOK: query: DROP TABLE accumulo_index_test
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE accumulo_index_test
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE accumulo_index_test (
- rowid string,
- active boolean,
- num_offices tinyint,
- num_personel smallint,
- total_manhours int,
- num_shareholders bigint,
- eff_rating float,
- err_rating double,
- yearly_production decimal,
- start_date date,
- address varchar(100),
- phone char(13),
- last_update timestamp )
-ROW FORMAT SERDE 'org.apache.hadoop.hive.accumulo.serde.AccumuloSerDe'
-STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
-WITH SERDEPROPERTIES (
- "accumulo.columns.mapping" = ":rowID,a:act,a:off,a:per,a:mhs,a:shs,a:eff,a:err,a:yp,a:sd,a:addr,a:ph,a:lu",
- "accumulo.table.name"="accumulo_index_test",
- "accumulo.indexed.columns"="*",
- "accumulo.indextable.name"="accumulo_index_idx"
- )
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@accumulo_index_test
-POSTHOOK: query: CREATE TABLE accumulo_index_test (
- rowid string,
- active boolean,
- num_offices tinyint,
- num_personel smallint,
- total_manhours int,
- num_shareholders bigint,
- eff_rating float,
- err_rating double,
- yearly_production decimal,
- start_date date,
- address varchar(100),
- phone char(13),
- last_update timestamp )
-ROW FORMAT SERDE 'org.apache.hadoop.hive.accumulo.serde.AccumuloSerDe'
-STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
-WITH SERDEPROPERTIES (
- "accumulo.columns.mapping" = ":rowID,a:act,a:off,a:per,a:mhs,a:shs,a:eff,a:err,a:yp,a:sd,a:addr,a:ph,a:lu",
- "accumulo.table.name"="accumulo_index_test",
- "accumulo.indexed.columns"="*",
- "accumulo.indextable.name"="accumulo_index_idx"
- )
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@accumulo_index_test
-PREHOOK: query: insert into accumulo_index_test values( "row1", true, 55, 107, 555555, 1223232332,
- 4.5, 0.8, 1232223, "2001-10-10", "123 main street",
- "555-555-5555", "2016-02-22 12:45:07.000000000")
-PREHOOK: type: QUERY
-PREHOOK: Output: default@accumulo_index_test
-POSTHOOK: query: insert into accumulo_index_test values( "row1", true, 55, 107, 555555, 1223232332,
- 4.5, 0.8, 1232223, "2001-10-10", "123 main street",
- "555-555-5555", "2016-02-22 12:45:07.000000000")
-POSTHOOK: type: QUERY
-POSTHOOK: Output: default@accumulo_index_test
-PREHOOK: query: select * from accumulo_index_test where active = 'true'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where active = 'true'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where num_offices = 55
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where num_offices = 55
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where num_personel = 107
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where num_personel = 107
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where total_manhours < 555556
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where total_manhours < 555556
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where num_shareholders >= 1223232331
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where num_shareholders >= 1223232331
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where eff_rating <= 4.5
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where eff_rating <= 4.5
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where err_rating >= 0.8
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where err_rating >= 0.8
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where yearly_production = 1232223
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where yearly_production = 1232223
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where start_date = "2001-10-10"
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where start_date = "2001-10-10"
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where address >= "100 main street"
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where address >= "100 main street"
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where phone <= "555-555-5555"
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where phone <= "555-555-5555"
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: select * from accumulo_index_test where last_update >= "2016-02-22 12:45:07"
-PREHOOK: type: QUERY
-PREHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-POSTHOOK: query: select * from accumulo_index_test where last_update >= "2016-02-22 12:45:07"
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@accumulo_index_test
-#### A masked pattern was here ####
-row1 true 55 107 555555 1223232332 4.5 0.8 1232223 2001-10-10 123 main street 555-555-5555 2016-02-22 12:45:07
-PREHOOK: query: DROP TABLE accumulo_index_test
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@accumulo_index_test
-PREHOOK: Output: default@accumulo_index_test
-POSTHOOK: query: DROP TABLE accumulo_index_test
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@accumulo_index_test
-POSTHOOK: Output: default@accumulo_index_test
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/pom.xml
----------------------------------------------------------------------
diff --git a/beeline/pom.xml b/beeline/pom.xml
index b0a9a0b..58ca92e 100644
--- a/beeline/pom.xml
+++ b/beeline/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/src/java/org/apache/hive/beeline/BeeLine.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index a589f33..3c8fccc 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -22,7 +22,6 @@
*/
package org.apache.hive.beeline;
-import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.EOFException;
@@ -30,7 +29,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.SequenceInputStream;
import java.lang.reflect.InvocationTargetException;
@@ -61,7 +59,6 @@ import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
@@ -151,10 +148,6 @@ public class BeeLine implements Closeable {
// Indicates if this instance of beeline is running in compatibility mode, or beeline mode
private boolean isBeeLine = true;
- // Indicates that we are in test mode.
- // Print only the errors, the operation log and the query results.
- private boolean isTestMode = false;
-
private static final Options options = new Options();
public static final String BEELINE_DEFAULT_JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver";
@@ -545,7 +538,6 @@ public class BeeLine implements Closeable {
public void run() {
try {
if (history != null) {
- history.setMaxSize(getOpts().getMaxHistoryRows());
history.flush();
}
} catch (IOException e) {
@@ -1387,55 +1379,6 @@ public class BeeLine implements Closeable {
return lineTrimmed.startsWith("#") || lineTrimmed.startsWith("--");
}
- String[] getCommands(File file) throws IOException {
- List<String> cmds = new LinkedList<String>();
- try (BufferedReader reader =
- new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"))) {
- StringBuilder cmd = null;
- while (true) {
- String scriptLine = reader.readLine();
-
- if (scriptLine == null) {
- break;
- }
-
- String trimmedLine = scriptLine.trim();
- if (getOpts().getTrimScripts()) {
- scriptLine = trimmedLine;
- }
-
- if (cmd != null) {
- // we're continuing an existing command
- cmd.append("\n");
- cmd.append(scriptLine);
- if (trimmedLine.endsWith(";")) {
- // this command has terminated
- cmds.add(cmd.toString());
- cmd = null;
- }
- } else {
- // we're starting a new command
- if (needsContinuation(scriptLine)) {
- // multi-line
- cmd = new StringBuilder(scriptLine);
- } else {
- // single-line
- cmds.add(scriptLine);
- }
- }
- }
-
- if (cmd != null) {
- // ### REVIEW: oops, somebody left the last command
- // unterminated; should we fix it for them or complain?
- // For now be nice and fix it.
- cmd.append(";");
- cmds.add(cmd.toString());
- }
- }
- return cmds.toArray(new String[0]);
- }
-
/**
* Print the specified message to the console
*
@@ -2442,19 +2385,4 @@ public class BeeLine implements Closeable {
public void setCurrentDatabase(String currentDatabase) {
this.currentDatabase = currentDatabase;
}
-
- /**
- * Setting the BeeLine into test mode.
- * Print only the errors, the operation log and the query results.
- * Should be used only by tests.
- *
- * @param isTestMode
- */
- void setIsTestMode(boolean isTestMode) {
- this.isTestMode = isTestMode;
- }
-
- boolean isTestMode() {
- return isTestMode;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
index f85d8a3..7e6846d 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
@@ -45,7 +45,6 @@ import jline.Terminal;
import jline.TerminalFactory;
import jline.console.completer.Completer;
import jline.console.completer.StringsCompleter;
-import jline.console.history.MemoryHistory;
import org.apache.hadoop.hive.conf.HiveConf;
class BeeLineOpts implements Completer {
@@ -62,7 +61,7 @@ class BeeLineOpts implements Completer {
public static final int DEFAULT_MAX_COLUMN_WIDTH = 50;
public static final int DEFAULT_INCREMENTAL_BUFFER_ROWS = 1000;
- public static final String URL_ENV_PREFIX = "BEELINE_URL_";
+ public static String URL_ENV_PREFIX = "BEELINE_URL_";
private final BeeLine beeLine;
private boolean autosave = false;
@@ -101,7 +100,6 @@ class BeeLineOpts implements Completer {
private final File rcFile = new File(saveDir(), "beeline.properties");
private String historyFile = new File(saveDir(), "history").getAbsolutePath();
- private int maxHistoryRows = MemoryHistory.DEFAULT_MAX_SIZE;
private String scriptFile = null;
private String[] initFiles = null;
@@ -433,17 +431,6 @@ class BeeLineOpts implements Completer {
return historyFile;
}
- /**
- * @param numRows - the number of rows to store in history file
- */
- public void setMaxHistoryRows(int numRows) {
- this.maxHistoryRows = numRows;
- }
-
- public int getMaxHistoryRows() {
- return maxHistoryRows;
- }
-
public void setScriptFile(String scriptFile) {
this.scriptFile = scriptFile;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/src/java/org/apache/hive/beeline/Commands.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java
index 08d53ca..99ee82c 100644
--- a/beeline/src/java/org/apache/hive/beeline/Commands.java
+++ b/beeline/src/java/org/apache/hive/beeline/Commands.java
@@ -55,6 +55,7 @@ import java.util.TreeSet;
import org.apache.hadoop.hive.common.cli.ShellCmdExecutor;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.conf.HiveVariableSource;
import org.apache.hadoop.hive.conf.SystemVariables;
import org.apache.hadoop.hive.conf.VariableSubstitution;
@@ -977,8 +978,7 @@ public class Commands {
hasResults = ((CallableStatement) stmnt).execute();
} else {
stmnt = beeLine.createStatement();
- // In test mode we want the operation logs regardless of the settings
- if (!beeLine.isTestMode() && beeLine.getOpts().isSilent()) {
+ if (beeLine.getOpts().isSilent()) {
hasResults = stmnt.execute(sql);
} else {
InPlaceUpdateStream.EventNotifier eventNotifier =
@@ -1221,61 +1221,46 @@ public class Commands {
if (entireLineAsCommand) {
cmdList.add(line);
} else {
- StringBuilder command = new StringBuilder();
+ StringBuffer command = new StringBuffer();
- // Marker to track if there is starting double quote without an ending double quote
boolean hasUnterminatedDoubleQuote = false;
+ boolean hasUntermindatedSingleQuote = false;
- // Marker to track if there is starting single quote without an ending double quote
- boolean hasUnterminatedSingleQuote = false;
-
- // Index of the last seen semicolon in the given line
int lastSemiColonIndex = 0;
char[] lineChars = line.toCharArray();
- // Marker to track if the previous character was an escape character
boolean wasPrevEscape = false;
-
int index = 0;
-
- // Iterate through the line and invoke the addCmdPart method whenever a semicolon is seen that is not inside a
- // quoted string
for (; index < lineChars.length; index++) {
switch (lineChars[index]) {
case '\'':
- // If a single quote is seen and the index is not inside a double quoted string and the previous character
- // was not an escape, then update the hasUnterminatedSingleQuote flag
if (!hasUnterminatedDoubleQuote && !wasPrevEscape) {
- hasUnterminatedSingleQuote = !hasUnterminatedSingleQuote;
+ hasUntermindatedSingleQuote = !hasUntermindatedSingleQuote;
}
wasPrevEscape = false;
break;
case '\"':
- // If a double quote is seen and the index is not inside a single quoted string and the previous character
- // was not an escape, then update the hasUnterminatedDoubleQuote flag
- if (!hasUnterminatedSingleQuote && !wasPrevEscape) {
+ if (!hasUntermindatedSingleQuote && !wasPrevEscape) {
hasUnterminatedDoubleQuote = !hasUnterminatedDoubleQuote;
}
wasPrevEscape = false;
break;
case ';':
- // If a semicolon is seen, and the line isn't inside a quoted string, then treat
- // line[lastSemiColonIndex] to line[index] as a single command
- if (!hasUnterminatedDoubleQuote && !hasUnterminatedSingleQuote) {
+ if (!hasUnterminatedDoubleQuote && !hasUntermindatedSingleQuote) {
addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index));
lastSemiColonIndex = index + 1;
}
wasPrevEscape = false;
break;
case '\\':
- wasPrevEscape = !wasPrevEscape;
+ wasPrevEscape = true;
break;
default:
wasPrevEscape = false;
break;
}
}
- // If the line doesn't end with a ; or if the line is empty, add the cmd part
+ // if the line doesn't end with a ; or if the line is empty, add the cmd part
if (lastSemiColonIndex != index || lineChars.length == 0) {
addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index));
}
@@ -1287,7 +1272,7 @@ public class Commands {
* Given a cmdpart (e.g. if a command spans multiple lines), add to the current command, and if
* applicable add that command to the {@link List} of commands
*/
- private void addCmdPart(List<String> cmdList, StringBuilder command, String cmdpart) {
+ private void addCmdPart(List<String> cmdList, StringBuffer command, String cmdpart) {
if (cmdpart.endsWith("\\")) {
command.append(cmdpart.substring(0, cmdpart.length() - 1)).append(";");
return;
@@ -1342,12 +1327,7 @@ public class Commands {
try {
List<String> queryLogs = hiveStatement.getQueryLog();
for (String log : queryLogs) {
- if (!commands.beeLine.isTestMode()) {
- commands.beeLine.info(log);
- } else {
- // In test mode print the logs to the output
- commands.beeLine.output(log);
- }
+ commands.beeLine.info(log);
}
if (!queryLogs.isEmpty()) {
notifier.operationLogShowedToUser();
@@ -1391,12 +1371,7 @@ public class Commands {
return;
}
for (String log : logs) {
- if (!beeLine.isTestMode()) {
- beeLine.info(log);
- } else {
- // In test mode print the logs to the output
- beeLine.output(log);
- }
+ beeLine.info(log);
}
} while (logs.size() > 0);
} else {
@@ -1798,10 +1773,60 @@ public class Commands {
return false;
}
+ List<String> cmds = new LinkedList<String>();
+
try {
- String[] cmds = beeLine.getCommands(new File(parts[1]));
+ BufferedReader reader = new BufferedReader(new FileReader(
+ parts[1]));
+ try {
+ // ### NOTE: fix for sf.net bug 879427
+ StringBuilder cmd = null;
+ for (;;) {
+ String scriptLine = reader.readLine();
+
+ if (scriptLine == null) {
+ break;
+ }
+
+ String trimmedLine = scriptLine.trim();
+ if (beeLine.getOpts().getTrimScripts()) {
+ scriptLine = trimmedLine;
+ }
+
+ if (cmd != null) {
+ // we're continuing an existing command
+ cmd.append(" \n");
+ cmd.append(scriptLine);
+ if (trimmedLine.endsWith(";")) {
+ // this command has terminated
+ cmds.add(cmd.toString());
+ cmd = null;
+ }
+ } else {
+ // we're starting a new command
+ if (beeLine.needsContinuation(scriptLine)) {
+ // multi-line
+ cmd = new StringBuilder(scriptLine);
+ } else {
+ // single-line
+ cmds.add(scriptLine);
+ }
+ }
+ }
+
+ if (cmd != null) {
+ // ### REVIEW: oops, somebody left the last command
+ // unterminated; should we fix it for them or complain?
+ // For now be nice and fix it.
+ cmd.append(";");
+ cmds.add(cmd.toString());
+ }
+ } finally {
+ reader.close();
+ }
+
// success only if all the commands were successful
- return beeLine.runCommands(cmds) == cmds.length;
+ return beeLine.runCommands(cmds) == cmds.size();
} catch (Exception e) {
return beeLine.error(e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
index 711f6a8..181f0d2 100644
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
+++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
@@ -292,7 +292,7 @@ public class HiveSchemaHelper {
// Derby commandline parser
public static class DerbyCommandParser extends AbstractCommandParser {
- private static final String DERBY_NESTING_TOKEN = "RUN";
+ private static String DERBY_NESTING_TOKEN = "RUN";
public DerbyCommandParser(String dbOpts, String msUsername, String msPassword,
HiveConf hiveConf) {
@@ -380,11 +380,11 @@ public class HiveSchemaHelper {
// Postgres specific parser
public static class PostgresCommandParser extends AbstractCommandParser {
- private static final String POSTGRES_NESTING_TOKEN = "\\i";
+ private static String POSTGRES_NESTING_TOKEN = "\\i";
@VisibleForTesting
- public static final String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings";
+ public static String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings";
@VisibleForTesting
- public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
+ public static String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
public PostgresCommandParser(String dbOpts, String msUsername, String msPassword,
HiveConf hiveConf) {
@@ -427,7 +427,7 @@ public class HiveSchemaHelper {
//Oracle specific parser
public static class OracleCommandParser extends AbstractCommandParser {
- private static final String ORACLE_NESTING_TOKEN = "@";
+ private static String ORACLE_NESTING_TOKEN = "@";
public OracleCommandParser(String dbOpts, String msUsername, String msPassword,
HiveConf hiveConf) {
@@ -451,7 +451,7 @@ public class HiveSchemaHelper {
//MSSQL specific parser
public static class MSSQLCommandParser extends AbstractCommandParser {
- private static final String MSSQL_NESTING_TOKEN = ":r";
+ private static String MSSQL_NESTING_TOKEN = ":r";
public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword,
HiveConf hiveConf) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
index 7dd4d5f..2c088c9 100644
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
+++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
@@ -89,7 +89,14 @@ public class HiveSchemaTool {
}
this.hiveConf = hiveConf;
this.dbType = dbType;
- this.metaStoreSchemaInfo = new MetaStoreSchemaInfo(hiveHome, dbType);
+ this.metaStoreSchemaInfo = new MetaStoreSchemaInfo(hiveHome, hiveConf, dbType);
+ userName = hiveConf.get(ConfVars.METASTORE_CONNECTION_USER_NAME.varname);
+ try {
+ passWord = ShimLoader.getHadoopShims().getPassword(hiveConf,
+ HiveConf.ConfVars.METASTOREPWD.varname);
+ } catch (IOException err) {
+ throw new HiveMetaException("Error getting metastore password", err);
+ }
}
public HiveConf getHiveConf() {
@@ -586,40 +593,29 @@ public class HiveSchemaTool {
}
public void doValidate() throws HiveMetaException {
- System.out.println("Starting metastore validation\n");
+ System.out.println("Starting metastore validation");
Connection conn = getConnectionToMetastore(false);
- boolean success = true;
try {
- if (validateSchemaVersions(conn)) {
+ if (validateSchemaVersions(conn))
System.out.println("[SUCCESS]\n");
- } else {
- success = false;
+ else
System.out.println("[FAIL]\n");
- }
- if (validateSequences(conn)) {
+ if (validateSequences(conn))
System.out.println("[SUCCESS]\n");
- } else {
- success = false;
+ else
System.out.println("[FAIL]\n");
- }
- if (validateSchemaTables(conn)) {
+ if (validateSchemaTables(conn))
System.out.println("[SUCCESS]\n");
- } else {
- success = false;
+ else
System.out.println("[FAIL]\n");
- }
- if (validateLocations(conn, this.validationServers)) {
+ if (validateLocations(conn, this.validationServers))
System.out.println("[SUCCESS]\n");
- } else {
- success = false;
+ else
System.out.println("[FAIL]\n");
- }
- if (validateColumnNullValues(conn)) {
+ if (validateColumnNullValues(conn))
System.out.println("[SUCCESS]\n");
- } else {
- success = false;
+ else
System.out.println("[FAIL]\n");
- }
} finally {
if (conn != null) {
try {
@@ -630,13 +626,7 @@ public class HiveSchemaTool {
}
}
- System.out.print("Done with metastore validation: ");
- if (!success) {
- System.out.println("[FAIL]");
- System.exit(1);
- } else {
- System.out.println("[SUCCESS]");
- }
+ System.out.println("Done with metastore validation");
}
boolean validateSequences(Connection conn) throws HiveMetaException {
@@ -729,14 +719,14 @@ public class HiveSchemaTool {
version = getMetaStoreSchemaVersion(hmsConn);
} catch (HiveMetaException he) {
System.err.println("Failed to determine schema version from Hive Metastore DB," + he.getMessage());
- LOG.debug("Failed to determine schema version from Hive Metastore DB," + he.getMessage());
+ LOG.error("Failed to determine schema version from Hive Metastore DB," + he.getMessage());
return false;
}
// re-open the hms connection
hmsConn = getConnectionToMetastore(false);
- LOG.debug("Validating tables in the schema for version " + version);
+ LOG.info("Validating tables in the schema for version " + version);
try {
metadata = conn.getMetaData();
String[] types = {"TABLE"};
@@ -770,7 +760,7 @@ public class HiveSchemaTool {
subScripts.addAll(findCreateTable(schemaFile, schemaTables));
while (subScripts.size() > 0) {
schemaFile = baseDir + "/" + dbType + "/" + subScripts.remove(0);
- LOG.debug("Parsing subscript " + schemaFile);
+ LOG.info("Parsing subscript " + schemaFile);
subScripts.addAll(findCreateTable(schemaFile, schemaTables));
}
} catch (Exception e) {
@@ -785,12 +775,13 @@ public class HiveSchemaTool {
int schemaSize = schemaTables.size();
schemaTables.removeAll(dbTables);
if (schemaTables.size() > 0) {
- System.out.println("Table(s) [ " + Arrays.toString(schemaTables.toArray())
+ System.out.println("Found " + schemaSize + " tables in schema definition, " +
+ schemaTables.size() + " tables [ " + Arrays.toString(schemaTables.toArray())
+ " ] are missing from the metastore database schema.");
System.out.println("Schema table validation failed!!!");
return false;
} else {
- System.out.println("Succeeded in schema table validation.");
+ System.out.println("Succeeded in schema table validation. " + schemaSize + " tables matched");
return true;
}
}
@@ -1111,19 +1102,9 @@ public class HiveSchemaTool {
if (line.hasOption("userName")) {
schemaTool.setUserName(line.getOptionValue("userName"));
- } else {
- schemaTool.setUserName(
- schemaTool.getHiveConf().get(ConfVars.METASTORE_CONNECTION_USER_NAME.varname));
}
if (line.hasOption("passWord")) {
schemaTool.setPassWord(line.getOptionValue("passWord"));
- } else {
- try {
- schemaTool.setPassWord(ShimLoader.getHadoopShims().getPassword(schemaTool.getHiveConf(),
- HiveConf.ConfVars.METASTOREPWD.varname));
- } catch (IOException err) {
- throw new HiveMetaException("Error getting metastore password", err);
- }
}
if (line.hasOption("dryRun")) {
schemaTool.setDryRun(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java
index 7d7d9ae..93a6231 100644
--- a/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java
+++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java
@@ -44,7 +44,7 @@ public class UserHS2ConnectionFileParser implements HS2ConnectionFileParser {
+ (System.getProperty("os.name").toLowerCase().indexOf("windows") != -1 ? "" : ".")
+ "beeline" + File.separator;
public static final String ETC_HIVE_CONF_LOCATION =
- File.separator + "etc" + File.separator + "hive" + File.separator + "conf";
+ File.separator + "etc" + File.separator + "conf" + File.separator + "hive";
private final List<String> locations = new ArrayList<>();
private static final Logger log = LoggerFactory.getLogger(UserHS2ConnectionFileParser.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/src/java/org/apache/hive/beeline/logs/BeelineInPlaceUpdateStream.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/logs/BeelineInPlaceUpdateStream.java b/beeline/src/java/org/apache/hive/beeline/logs/BeelineInPlaceUpdateStream.java
index 40cde0c..51344e3 100644
--- a/beeline/src/java/org/apache/hive/beeline/logs/BeelineInPlaceUpdateStream.java
+++ b/beeline/src/java/org/apache/hive/beeline/logs/BeelineInPlaceUpdateStream.java
@@ -1,20 +1,3 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
package org.apache.hive.beeline.logs;
import org.apache.hadoop.hive.common.log.InPlaceUpdate;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/src/main/resources/BeeLine.properties
----------------------------------------------------------------------
diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties
index 7011221..e33b812 100644
--- a/beeline/src/main/resources/BeeLine.properties
+++ b/beeline/src/main/resources/BeeLine.properties
@@ -202,7 +202,6 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \
\ --delimiterForDSV=DELIMITER specify the delimiter for delimiter-separated values output format (default: |)\n \
\ --isolation=LEVEL set the transaction isolation level\n \
\ --nullemptystring=[true/false] set to true to get historic behavior of printing null as empty string\n \
-\ --maxHistoryRows=MAXHISTORYROWS The maximum number of rows to store beeline history.\n \
\ --help display this message\n \
\n \
\ Example:\n \
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
----------------------------------------------------------------------
diff --git a/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java b/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
index 2884cc8..d73d374 100644
--- a/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
+++ b/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
@@ -319,16 +319,4 @@ public class TestBeelineArgParsing {
Assert.assertTrue(bl.properties.get(0).equals("props"));
bl.close();
}
-
- /**
- * Test maxHistoryRows parameter option.
- */
- @Test
- public void testMaxHistoryRows() throws Exception {
- TestBeeline bl = new TestBeeline();
- String args[] = new String[] {"--maxHistoryRows=100"};
- Assert.assertEquals(0, bl.initArgs(args));
- Assert.assertTrue(bl.getOpts().getMaxHistoryRows() == 100);
- bl.close();
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java b/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
index 4cd5124..8d386da 100644
--- a/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
+++ b/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
@@ -1,20 +1,3 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
package org.apache.hive.beeline;
import org.apache.hadoop.hive.conf.HiveConf;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/cli/pom.xml
----------------------------------------------------------------------
diff --git a/cli/pom.xml b/cli/pom.xml
index 71d214b..10fb1b9 100644
--- a/cli/pom.xml
+++ b/cli/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java
----------------------------------------------------------------------
diff --git a/cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java b/cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java
index 24550fa..f1806a0 100644
--- a/cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java
+++ b/cli/src/java/org/apache/hadoop/hive/cli/RCFileCat.java
@@ -54,7 +54,7 @@ public class RCFileCat implements Tool{
// In verbose mode, print an update per RECORD_PRINT_INTERVAL records
private static final int RECORD_PRINT_INTERVAL = (1024*1024);
- protected boolean test = false;
+ protected static boolean test=false;
public RCFileCat() {
super();
@@ -63,12 +63,12 @@ public class RCFileCat implements Tool{
onUnmappableCharacter(CodingErrorAction.REPLACE);
}
- private CharsetDecoder decoder;
+ private static CharsetDecoder decoder;
Configuration conf = null;
- private static final String TAB ="\t";
- private static final String NEWLINE ="\r\n";
+ private static String TAB ="\t";
+ private static String NEWLINE ="\r\n";
@Override
public int run(String[] args) throws Exception {
@@ -243,7 +243,7 @@ public class RCFileCat implements Tool{
this.conf = conf;
}
- private static final String Usage = "RCFileCat [--start=start_offet] [--length=len] [--verbose] " +
+ private static String Usage = "RCFileCat [--start=start_offet] [--length=len] [--verbose] " +
"[--column-sizes | --column-sizes-pretty] [--file-sizes] fileName";
public static void main(String[] args) {
@@ -262,7 +262,7 @@ public class RCFileCat implements Tool{
}
}
- private void setupBufferedOutput() {
+ private static void setupBufferedOutput() {
OutputStream pdataOut;
if (test) {
pdataOut = System.out;
@@ -275,7 +275,6 @@ public class RCFileCat implements Tool{
new PrintStream(bos, false);
System.setOut(ps);
}
-
private static void printUsage(String errorMsg) {
System.err.println(Usage);
if(errorMsg != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java
----------------------------------------------------------------------
diff --git a/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java b/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java
index 4cb4a19..11ceb31 100644
--- a/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java
+++ b/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java
@@ -25,6 +25,8 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
+import java.net.URI;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -75,7 +77,7 @@ public class TestRCFileCat {
writer.close();
RCFileCat fileCat = new RCFileCat();
- fileCat.test=true;
+ RCFileCat.test=true;
fileCat.setConf(new Configuration());
// set fake input and output streams
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index e6722ba..8474a87 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -69,24 +69,20 @@
<version>${jline.version}</version>
</dependency>
<dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-rewrite</artifactId>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>jetty-all</artifactId>
+ <version>${jetty.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-webapp</artifactId>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ <version>${javax-servlet.version}</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
@@ -133,18 +129,6 @@
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
@@ -165,10 +149,6 @@
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/common/src/java/org/apache/hadoop/hive/common/CopyOnFirstWriteProperties.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/CopyOnFirstWriteProperties.java b/common/src/java/org/apache/hadoop/hive/common/CopyOnFirstWriteProperties.java
deleted file mode 100644
index d4d078b..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/CopyOnFirstWriteProperties.java
+++ /dev/null
@@ -1,344 +0,0 @@
-package org.apache.hadoop.hive.common;
-
-import com.google.common.collect.Interner;
-import com.google.common.collect.Interners;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.io.PrintWriter;
-import java.io.Reader;
-import java.lang.reflect.Field;
-import java.util.Collection;
-import java.util.Enumeration;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-
-/**
- * A special subclass of Properties, designed to save memory when many identical
- * copies of Properties would otherwise be created. To achieve that, we use the
- * 'interned' field, which points to the same Properties object for all instances
- * of CopyOnFirstWriteProperties that were created with identical contents.
- * However, as soon as any mutating method is called, contents are copied from
- * the 'interned' properties into this instance.
- */
-public class CopyOnFirstWriteProperties extends Properties {
-
- private Properties interned;
-
- private static Interner<Properties> INTERNER = Interners.newWeakInterner();
- private static Field defaultsField;
- static {
- try {
- defaultsField = Properties.class.getDeclaredField("defaults");
- defaultsField.setAccessible(true);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- public CopyOnFirstWriteProperties(Properties p) {
- setInterned(p);
- }
-
- /************* Public API of java.util.Properties ************/
-
- @Override
- public String getProperty(String key) {
- if (interned != null) return interned.getProperty(key);
- else return super.getProperty(key);
- }
-
- @Override
- public String getProperty(String key, String defaultValue) {
- if (interned != null) return interned.getProperty(key, defaultValue);
- else return super.getProperty(key, defaultValue);
- }
-
- @Override
- public void list(PrintStream out) {
- if (interned != null) interned.list(out);
- else super.list(out);
- }
-
- @Override
- public void list(PrintWriter out) {
- if (interned != null) interned.list(out);
- else super.list(out);
- }
-
- @Override
- public synchronized void load(InputStream inStream) throws IOException {
- if (interned != null) copyFromInternedToThis();
- super.load(inStream);
- }
-
- @Override
- public synchronized void load(Reader reader) throws IOException {
- if (interned != null) copyFromInternedToThis();
- super.load(reader);
- }
-
- @Override
- public synchronized void loadFromXML(InputStream inStream) throws IOException {
- if (interned != null) copyFromInternedToThis();
- super.loadFromXML(inStream);
- }
-
- @Override
- public Enumeration<?> propertyNames() {
- if (interned != null) return interned.propertyNames();
- else return super.propertyNames();
- }
-
- @Override
- public synchronized Object setProperty(String key, String value) {
- if (interned != null) copyFromInternedToThis();
- return super.setProperty(key, value);
- }
-
- @Override
- public void store(OutputStream out, String comments) throws IOException {
- if (interned != null) interned.store(out, comments);
- else super.store(out, comments);
- }
-
- @Override
- public void storeToXML(OutputStream os, String comment) throws IOException {
- if (interned != null) interned.storeToXML(os, comment);
- else super.storeToXML(os, comment);
- }
-
- @Override
- public void storeToXML(OutputStream os, String comment, String encoding)
- throws IOException {
- if (interned != null) interned.storeToXML(os, comment, encoding);
- else super.storeToXML(os, comment, encoding);
- }
-
- @Override
- public Set<String> stringPropertyNames() {
- if (interned != null) return interned.stringPropertyNames();
- else return super.stringPropertyNames();
- }
-
- /************* Public API of java.util.Hashtable ************/
-
- @Override
- public synchronized void clear() {
- if (interned != null) copyFromInternedToThis();
- super.clear();
- }
-
- @Override
- public synchronized Object clone() {
- if (interned != null) return new CopyOnFirstWriteProperties(interned);
- else return super.clone();
- }
-
- @Override
- public synchronized Object compute(Object key, BiFunction remappingFunction) {
- if (interned != null) copyFromInternedToThis(); // We do this because if function returns null,
- // the mapping for key is removed, i.e. the table is mutated.
- return super.compute(key, remappingFunction);
- }
-
- @Override
- public synchronized Object computeIfAbsent(Object key, Function mappingFunction) {
- if (interned != null) copyFromInternedToThis();
- return super.computeIfAbsent(key, mappingFunction);
- }
-
- @Override
- public synchronized Object computeIfPresent(Object key, BiFunction remappingFunction) {
- if (interned != null) copyFromInternedToThis();
- return super.computeIfPresent(key, remappingFunction);
- }
-
- @Override
- public synchronized boolean contains(Object value) {
- if (interned != null) return interned.contains(value);
- else return super.contains(value);
- }
-
- @Override
- public synchronized boolean containsKey(Object key) {
- if (interned != null) return interned.containsKey(key);
- else return super.containsKey(key);
- }
-
- @Override
- public synchronized boolean containsValue(Object value) {
- if (interned != null) return interned.containsValue(value);
- else return super.containsValue(value);
- }
-
- @Override
- public synchronized Enumeration<Object> elements() {
- if (interned != null) return interned.elements();
- else return super.elements();
- }
-
- @Override
- public Set<Map.Entry<Object, Object>> entrySet() {
- if (interned != null) return interned.entrySet();
- else return super.entrySet();
- }
-
- @Override
- public synchronized boolean equals(Object o) {
- if (interned != null) return interned.equals(o);
- else return super.equals(o);
- }
-
- @Override
- public synchronized void forEach(BiConsumer action) {
- if (interned != null) interned.forEach(action);
- else super.forEach(action);
- }
-
- @Override
- public synchronized Object get(Object key) {
- if (interned != null) return interned.get(key);
- else return super.get(key);
- }
-
- @Override
- public synchronized Object getOrDefault(Object key, Object defaultValue) {
- if (interned != null) return interned.getOrDefault(key, defaultValue);
- else return super.getOrDefault(key, defaultValue);
- }
-
- @Override
- public synchronized int hashCode() {
- if (interned != null) return interned.hashCode();
- else return super.hashCode();
- }
-
- @Override
- public synchronized boolean isEmpty() {
- if (interned != null) return interned.isEmpty();
- else return super.isEmpty();
- }
-
- @Override
- public synchronized Enumeration<Object> keys() {
- if (interned != null) return interned.keys();
- else return super.keys();
- }
-
- @Override
- public Set<Object> keySet() {
- if (interned != null) return interned.keySet();
- else return super.keySet();
- }
-
- @Override
- public synchronized Object merge(Object key, Object value, BiFunction remappingFunction) {
- if (interned != null) copyFromInternedToThis();
- return super.merge(key, value, remappingFunction);
- }
-
- @Override
- public synchronized Object put(Object key, Object value) {
- if (interned != null) copyFromInternedToThis();
- return super.put(key, value);
- }
-
- @Override
- public synchronized void putAll(Map<? extends Object, ? extends Object> t) {
- if (interned != null) copyFromInternedToThis();
- super.putAll(t);
- }
-
- @Override
- public synchronized Object putIfAbsent(Object key, Object value) {
- if (interned != null) copyFromInternedToThis();
- return super.putIfAbsent(key, value);
- }
-
- @Override
- public synchronized Object remove(Object key) {
- if (interned != null) copyFromInternedToThis();
- return super.remove(key);
- }
-
- @Override
- public synchronized boolean remove(Object key, Object value) {
- if (interned != null) copyFromInternedToThis();
- return super.remove(key, value);
- }
-
- @Override
- public synchronized Object replace(Object key, Object value) {
- if (interned != null) copyFromInternedToThis();
- return super.replace(key, value);
- }
-
- @Override
- public synchronized boolean replace(Object key, Object oldValue, Object newValue) {
- if (interned != null) copyFromInternedToThis();
- return super.replace(key, oldValue, newValue);
- }
-
- @Override
- public synchronized void replaceAll(BiFunction function) {
- if (interned != null) copyFromInternedToThis();
- super.replaceAll(function);
- }
-
- @Override
- public synchronized int size() {
- if (interned != null) return interned.size();
- else return super.size();
- }
-
- @Override
- public synchronized String toString() {
- if (interned != null) return interned.toString();
- else return super.toString();
- }
-
- @Override
- public Collection<Object> values() {
- if (interned != null) return interned.values();
- else return super.values();
- }
-
- /************* Private implementation ************/
-
- private void copyFromInternedToThis() {
- for (Map.Entry<?,?> e : interned.entrySet()) {
- super.put(e.getKey(), e.getValue());
- }
- try {
- // Unfortunately, we cannot directly read a protected field of non-this object
- this.defaults = (Properties) defaultsField.get(interned);
- } catch (IllegalAccessException e) { // Shouldn't happen
- throw new RuntimeException(e);
- }
- setInterned(null);
- }
-
- public void setInterned(Properties p) {
- if (p != null) {
- this.interned = INTERNER.intern(p);
- } else {
- this.interned = p;
- }
- }
-
- // These methods are required by serialization
-
- public CopyOnFirstWriteProperties() {
- }
-
- public Properties getInterned() {
- return interned;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index 0f7401c..c6bc9b9 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -25,8 +25,6 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.security.AccessControlException;
import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.BitSet;
import java.util.Collection;
import java.util.HashSet;
@@ -359,12 +357,6 @@ public final class FileUtils {
return getPathOrParentThatExists(fs, parentPath);
}
- public static void checkFileAccessWithImpersonation(final FileSystem fs, final FileStatus stat,
- final FsAction action, final String user)
- throws IOException, AccessControlException, InterruptedException, Exception {
- checkFileAccessWithImpersonation(fs, stat, action, user, null);
- }
-
/**
* Perform a check to determine if the user is able to access the file passed in.
* If the user name passed in is different from the current user, this method will
@@ -379,15 +371,13 @@ public final class FileUtils {
* check will be performed within a doAs() block to use the access privileges
* of this user. In this case the user must be configured to impersonate other
* users, otherwise this check will fail with error.
- * @param children List of children to be collected. If this is null, no children are collected.
- * To be set only if this is a directory
* @throws IOException
* @throws AccessControlException
* @throws InterruptedException
* @throws Exception
*/
public static void checkFileAccessWithImpersonation(final FileSystem fs,
- final FileStatus stat, final FsAction action, final String user, final List<FileStatus> children)
+ final FileStatus stat, final FsAction action, final String user)
throws IOException, AccessControlException, InterruptedException, Exception {
UserGroupInformation ugi = Utils.getUGI();
String currentUser = ugi.getShortUserName();
@@ -395,7 +385,6 @@ public final class FileUtils {
if (user == null || currentUser.equals(user)) {
// No need to impersonate user, do the checks as the currently configured user.
ShimLoader.getHadoopShims().checkFileAccess(fs, stat, action);
- addChildren(fs, stat.getPath(), children);
return;
}
@@ -408,7 +397,6 @@ public final class FileUtils {
public Object run() throws Exception {
FileSystem fsAsUser = FileSystem.get(fs.getUri(), fs.getConf());
ShimLoader.getHadoopShims().checkFileAccess(fsAsUser, stat, action);
- addChildren(fsAsUser, stat.getPath(), children);
return null;
}
});
@@ -417,20 +405,6 @@ public final class FileUtils {
}
}
- private static void addChildren(FileSystem fsAsUser, Path path, List<FileStatus> children)
- throws IOException {
- if (children != null) {
- FileStatus[] listStatus;
- try {
- listStatus = fsAsUser.listStatus(path);
- } catch (IOException e) {
- LOG.warn("Unable to list files under " + path + " : " + e);
- throw e;
- }
- children.addAll(Arrays.asList(listStatus));
- }
- }
-
/**
* Check if user userName has permissions to perform the given FsAction action
* on all files under the file whose FileStatus fileStatus is provided
@@ -457,26 +431,20 @@ public final class FileUtils {
dirActionNeeded.and(FsAction.EXECUTE);
}
- List<FileStatus> subDirsToCheck = null;
- if (isDir && recurse) {
- subDirsToCheck = new ArrayList<FileStatus>();
- }
-
try {
- checkFileAccessWithImpersonation(fs, fileStatus, action, userName, subDirsToCheck);
+ checkFileAccessWithImpersonation(fs, fileStatus, action, userName);
} catch (AccessControlException err) {
// Action not permitted for user
- LOG.warn("Action " + action + " denied on " + fileStatus.getPath() + " for user " + userName);
return false;
}
- if (subDirsToCheck == null || subDirsToCheck.isEmpty()) {
+ if ((!isDir) || (!recurse)) {
// no sub dirs to be checked
return true;
}
-
// check all children
- for (FileStatus childStatus : subDirsToCheck) {
+ FileStatus[] childStatuses = fs.listStatus(fileStatus.getPath());
+ for (FileStatus childStatus : childStatuses) {
// check children recursively - recurse is true if we're here.
if (!isActionPermittedForFileHierarchy(fs, childStatus, userName, action, true)) {
return false;
@@ -518,30 +486,11 @@ public final class FileUtils {
return false;
}
public static boolean isOwnerOfFileHierarchy(FileSystem fs, FileStatus fileStatus, String userName)
- throws IOException, InterruptedException {
+ throws IOException {
return isOwnerOfFileHierarchy(fs, fileStatus, userName, true);
}
- public static boolean isOwnerOfFileHierarchy(final FileSystem fs,
- final FileStatus fileStatus, final String userName, final boolean recurse)
- throws IOException, InterruptedException {
- UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(userName,
- UserGroupInformation.getLoginUser());
- try {
- boolean isOwner = proxyUser.doAs(new PrivilegedExceptionAction<Boolean>() {
- @Override
- public Boolean run() throws Exception {
- FileSystem fsAsUser = FileSystem.get(fs.getUri(), fs.getConf());
- return checkIsOwnerOfFileHierarchy(fsAsUser, fileStatus, userName, recurse);
- }
- });
- return isOwner;
- } finally {
- FileSystem.closeAllForUGI(proxyUser);
- }
- }
-
- public static boolean checkIsOwnerOfFileHierarchy(FileSystem fs, FileStatus fileStatus,
+ public static boolean isOwnerOfFileHierarchy(FileSystem fs, FileStatus fileStatus,
String userName, boolean recurse)
throws IOException {
if (!fileStatus.getOwner().equals(userName)) {
@@ -556,24 +505,59 @@ public final class FileUtils {
FileStatus[] childStatuses = fs.listStatus(fileStatus.getPath());
for (FileStatus childStatus : childStatuses) {
// check children recursively - recurse is true if we're here.
- if (!checkIsOwnerOfFileHierarchy(fs, childStatus, userName, true)) {
+ if (!isOwnerOfFileHierarchy(fs, childStatus, userName, true)) {
return false;
}
}
return true;
}
+ public static boolean mkdir(FileSystem fs, Path f, Configuration conf) throws IOException {
+ boolean inheritPerms = HiveConf.getBoolVar(conf, ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
+ return mkdir(fs, f, inheritPerms, conf);
+ }
+
/**
* Creates the directory and all necessary parent directories.
* @param fs FileSystem to use
* @param f path to create.
+ * @param inheritPerms whether directory inherits the permission of the last-existing parent path
* @param conf Hive configuration
* @return true if directory created successfully. False otherwise, including if it exists.
* @throws IOException exception in creating the directory
*/
- public static boolean mkdir(FileSystem fs, Path f, Configuration conf) throws IOException {
+ public static boolean mkdir(FileSystem fs, Path f, boolean inheritPerms, Configuration conf) throws IOException {
LOG.info("Creating directory if it doesn't exist: " + f);
- return fs.mkdirs(f);
+ if (!inheritPerms) {
+ //just create the directory
+ return fs.mkdirs(f);
+ } else {
+ //Check if the directory already exists. We want to change the permission
+ //to that of the parent directory only for newly created directories.
+ try {
+ return fs.getFileStatus(f).isDir();
+ } catch (FileNotFoundException ignore) {
+ }
+ //inherit perms: need to find last existing parent path, and apply its permission on entire subtree.
+ Path lastExistingParent = f;
+ Path firstNonExistentParent = null;
+ while (!fs.exists(lastExistingParent)) {
+ firstNonExistentParent = lastExistingParent;
+ lastExistingParent = lastExistingParent.getParent();
+ }
+ boolean success = fs.mkdirs(f);
+ if (!success) {
+ return false;
+ } else {
+ //set on the entire subtree
+ if (inheritPerms) {
+ HdfsUtils.setFullFileStatus(conf,
+ new HdfsUtils.HadoopFileStatus(conf, fs, lastExistingParent), fs,
+ firstNonExistentParent, true);
+ }
+ return true;
+ }
+ }
}
public static Path makeAbsolute(FileSystem fileSystem, Path path) throws IOException {
@@ -626,6 +610,11 @@ public final class FileUtils {
if (!triedDistcp) {
copied = FileUtil.copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf);
}
+
+ boolean inheritPerms = conf.getBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
+ if (copied && inheritPerms) {
+ HdfsUtils.setFullFileStatus(conf, new HdfsUtils.HadoopFileStatus(conf, dstFS, dst.getParent()), dstFS, dst, true);
+ }
return copied;
}
@@ -637,19 +626,15 @@ public final class FileUtils {
* @return true if move successful
* @throws IOException
*/
- public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf, boolean purge)
+ public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf)
throws IOException {
LOG.debug("deleting " + f);
boolean result = false;
try {
- if(purge) {
- LOG.debug("purge is set to true. Not moving to Trash " + f);
- } else {
- result = Trash.moveToAppropriateTrash(fs, f, conf);
- if (result) {
- LOG.trace("Moved to trash: " + f);
- return true;
- }
+ result = Trash.moveToAppropriateTrash(fs, f, conf);
+ if (result) {
+ LOG.trace("Moved to trash: " + f);
+ return true;
}
} catch (IOException ioe) {
// for whatever failure reason including that trash has lower encryption zone
@@ -661,11 +646,13 @@ public final class FileUtils {
if (!result) {
LOG.error("Failed to delete " + f);
}
+
return result;
}
- public static boolean rename(FileSystem fs, Path sourcePath,
- Path destPath, Configuration conf) throws IOException {
+ public static boolean renameWithPerms(FileSystem fs, Path sourcePath,
+ Path destPath, boolean inheritPerms,
+ Configuration conf) throws IOException {
LOG.info("Renaming " + sourcePath + " to " + destPath);
// If destPath directory exists, rename call will move the sourcePath
@@ -674,7 +661,20 @@ public final class FileUtils {
throw new IOException("Cannot rename the source path. The destination "
+ "path already exists.");
}
- return fs.rename(sourcePath, destPath);
+
+ if (!inheritPerms) {
+ //just rename the directory
+ return fs.rename(sourcePath, destPath);
+ } else {
+ //rename the directory
+ if (fs.rename(sourcePath, destPath)) {
+ HdfsUtils.setFullFileStatus(conf, new HdfsUtils.HadoopFileStatus(conf, fs, destPath.getParent()), fs, destPath,
+ true);
+ return true;
+ }
+
+ return false;
+ }
}
/**