You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by le...@apache.org on 2020/06/11 17:14:59 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-1148] improve
hive test coverage
This is an automated email from the ASF dual-hosted git repository.
lesun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 6f074c3 [GOBBLIN-1148] improve hive test coverage
6f074c3 is described below
commit 6f074c31128d443e7b67f4ca3673e4af7b5dcf05
Author: Hanghang Liu <ha...@Hanghangs-MacBook-Pro.local>
AuthorDate: Thu Jun 11 10:14:42 2020 -0700
[GOBBLIN-1148] improve hive test coverage
add simple hive spec test
add test get hive table
add hive snapshot registration policy test
Add Apache License header; Use json instead of
avro for testing
add apach license header
update method name
use .gitignore file to maintain a dummy directory
for hive snapshot testing
Closes #2990 from
hanghangliu/GOBBLIN-1148-improve-hive-test-
coverage
---
.../hive/metastore/HiveMetaStoreUtilsTest.java | 53 +++++++++++-
.../policy/HiveRegistrationPolicyBaseTest.java | 3 +-
.../policy/HiveSnapshotRegistrationPolicyTest.java | 70 ++++++++++++++++
.../gobblin/hive/spec/SimpleHiveSpecTest.java | 94 ++++++++++++++++++++++
.../resources/test-hive-table/snapshot1/.gitignore | 5 ++
5 files changed, 220 insertions(+), 5 deletions(-)
diff --git a/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtilsTest.java b/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtilsTest.java
index 38d5c65..2f3d5eb 100644
--- a/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtilsTest.java
+++ b/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtilsTest.java
@@ -17,13 +17,15 @@
package org.apache.gobblin.hive.metastore;
+import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.gobblin.hive.HiveRegistrationUnit;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat;
import org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
@@ -36,6 +38,8 @@ import org.testng.annotations.Test;
import org.apache.gobblin.configuration.State;
import org.apache.gobblin.hive.HiveTable;
+import static org.apache.gobblin.hive.metastore.HiveMetaStoreUtils.getParameters;
+
public class HiveMetaStoreUtilsTest {
@Test
@@ -165,4 +169,47 @@ public class HiveMetaStoreUtilsTest {
Assert.assertFalse(e instanceof NoSuchMethodException);
}
}
+
+ @Test
+ public void testGetHiveTable() throws Exception {
+ final String databaseName = "testdb";
+ final String tableName = "testtable";
+ final String tableSdLoc = "/tmp/testtable";
+ final String partitionName = "partitionName";
+
+ State serdeProps = new State();
+ serdeProps.setProp("avro.schema.literal", "{\"type\": \"record\", \"name\": \"TestEvent\","
+ + " \"namespace\": \"test.namespace\", \"fields\": [{\"name\":\"testName\"," + " \"type\": \"int\"}]}");
+
+ List<FieldSchema> fieldSchemas = new ArrayList<>();
+ fieldSchemas.add(new FieldSchema("testName","int", "testContent"));
+ SerDeInfo si = new SerDeInfo();
+ si.setParameters(getParameters(serdeProps));
+ si.setName(tableName);
+
+ StorageDescriptor sd = new StorageDescriptor(fieldSchemas, tableSdLoc,
+ AvroContainerInputFormat.class.getName(), AvroContainerOutputFormat.class.getName(),
+ false, 0, si, null, Lists.<Order>newArrayList(), null);
+ sd.setParameters(getParameters(serdeProps));
+ Table table = new Table(tableName, databaseName, "testOwner", 0, 0, 0, sd,
+ Lists.<FieldSchema>newArrayList(), Maps.<String,String>newHashMap(), "", "", "");
+ table.addToPartitionKeys(new FieldSchema(partitionName, "string", "some comment"));
+
+ HiveTable hiveTable = HiveMetaStoreUtils.getHiveTable(table);
+ Assert.assertEquals(hiveTable.getDbName(), databaseName);
+ Assert.assertEquals(hiveTable.getTableName(), tableName);
+
+ Assert.assertTrue(hiveTable.getInputFormat().isPresent());
+ Assert.assertTrue(hiveTable.getOutputFormat().isPresent());
+ Assert.assertEquals(hiveTable.getInputFormat().get(), AvroContainerInputFormat.class.getName());
+ Assert.assertEquals(hiveTable.getOutputFormat().get(), AvroContainerOutputFormat.class.getName());
+ Assert.assertNotNull(hiveTable.getSerDeType());
+
+ List<HiveRegistrationUnit.Column> fields = hiveTable.getColumns();
+ Assert.assertTrue(fields != null && fields.size() == 1);
+ HiveRegistrationUnit.Column fieldA = fields.get(0);
+ Assert.assertEquals(fieldA.getName(), "testName");
+ Assert.assertEquals(fieldA.getType(), "int");
+
+ }
}
diff --git a/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/policy/HiveRegistrationPolicyBaseTest.java b/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/policy/HiveRegistrationPolicyBaseTest.java
index 95de98a..d1f3812 100644
--- a/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/policy/HiveRegistrationPolicyBaseTest.java
+++ b/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/policy/HiveRegistrationPolicyBaseTest.java
@@ -156,8 +156,7 @@ public class HiveRegistrationPolicyBaseTest {
Assert.assertEquals(resultTable, "topic");
}
-
- private static void examine(HiveSpec spec, String dbName, String tableName) {
+ static void examine(HiveSpec spec, String dbName, String tableName) {
Assert.assertEquals(spec.getClass(), SimpleHiveSpec.class);
Assert.assertEquals(spec.getTable().getDbName(), dbName);
Assert.assertEquals(spec.getTable().getTableName(), tableName);
diff --git a/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/policy/HiveSnapshotRegistrationPolicyTest.java b/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/policy/HiveSnapshotRegistrationPolicyTest.java
new file mode 100644
index 0000000..8a7d507
--- /dev/null
+++ b/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/policy/HiveSnapshotRegistrationPolicyTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.hive.policy;
+
+import org.apache.gobblin.configuration.State;
+import org.apache.gobblin.hive.spec.HiveSpec;
+import org.apache.hadoop.fs.Path;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+
+import static org.apache.gobblin.hive.policy.HiveRegistrationPolicyBaseTest.examine;
+
+@Test(groups = {"gobblin.hive"})
+public class HiveSnapshotRegistrationPolicyTest {
+ private Path path;
+
+ @Test
+ public void testGetHiveSpecs()
+ throws IOException {
+ State state = new State();
+ this.path = new Path(getClass().getResource("/test-hive-table/snapshot1").toString());
+ //Test when directory contain zero snapshot
+ Collection<HiveSpec> specs = new HiveSnapshotRegistrationPolicy(state).getHiveSpecs(this.path);
+ Assert.assertEquals(specs.size(), 0);
+
+ //Test when directory contain snapshots sub-directory
+ this.path = new Path(getClass().getResource("/test-hive-table/").toString());
+
+ Assert.assertEquals(specs.size(), 0);
+ state.appendToListProp(HiveRegistrationPolicyBase.HIVE_DATABASE_NAME, "db1");
+ state.appendToListProp(HiveRegistrationPolicyBase.ADDITIONAL_HIVE_DATABASE_NAMES, "db2");
+
+ state.appendToListProp(HiveRegistrationPolicyBase.HIVE_TABLE_NAME, "tbl1");
+ state.appendToListProp(HiveRegistrationPolicyBase.ADDITIONAL_HIVE_TABLE_NAMES, "tbl2,tbl3");
+ specs = new HiveSnapshotRegistrationPolicy(state).getHiveSpecs(this.path);
+ Assert.assertEquals(specs.size(), 6);
+ Iterator<HiveSpec> iterator = specs.iterator();
+ HiveSpec spec = iterator.next();
+ examine(spec, "db1", "tbl1");
+ spec = iterator.next();
+ examine(spec, "db1", "tbl2");
+ spec = iterator.next();
+ examine(spec, "db1", "tbl3");
+ spec = iterator.next();
+ examine(spec, "db2", "tbl1");
+ spec = iterator.next();
+ examine(spec, "db2", "tbl2");
+ spec = iterator.next();
+ examine(spec, "db2", "tbl3");
+ }
+}
diff --git a/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/spec/SimpleHiveSpecTest.java b/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/spec/SimpleHiveSpecTest.java
new file mode 100644
index 0000000..ceffe9d
--- /dev/null
+++ b/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/spec/SimpleHiveSpecTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.hive.spec;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Optional;
+import org.apache.gobblin.hive.HiveTable;
+import org.apache.gobblin.hive.spec.activity.DropPartitionActivity;
+import org.apache.gobblin.hive.spec.activity.DropTableActivity;
+import org.apache.gobblin.hive.spec.predicate.PartitionNotExistPredicate;
+import org.apache.gobblin.hive.spec.predicate.TableNotExistPredicate;
+import org.apache.hadoop.fs.Path;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+
+@Test(groups = {"gobblin.hive"})
+public class SimpleHiveSpecTest {
+ private final String dbName = "db";
+ private final String tableName = "tbl";
+ private final String pathString = "tbl";
+ private SimpleHiveSpec simpleHiveSpec;
+
+ @BeforeClass
+ public void setup(){
+ HiveTable.Builder tableBuilder = new HiveTable.Builder();
+ tableBuilder.withDbName(dbName).withTableName(tableName);
+ HiveTable hiveTable = tableBuilder.build();
+
+ SimpleHiveSpec.Builder specBuilder = new SimpleHiveSpec.Builder(new Path(pathString))
+ .withPartition(Optional.absent())
+ .withTable(hiveTable);
+ simpleHiveSpec = specBuilder.build();
+ }
+
+ @Test(priority=1)
+ public void testBuildSimpleSpec() {
+ Assert.assertEquals(simpleHiveSpec.getTable().getDbName(), dbName);
+ Assert.assertEquals(simpleHiveSpec.getTable().getTableName(), tableName);
+ Assert.assertEquals(0, simpleHiveSpec.getPostActivities().size());
+ Assert.assertEquals(0, simpleHiveSpec.getPreActivities().size());
+ Assert.assertEquals(0, simpleHiveSpec.getPredicates().size());
+ Assert.assertEquals(Optional.absent(), simpleHiveSpec.getPartition());
+
+ String actualString = Objects.toStringHelper(simpleHiveSpec).omitNullValues().add("path", pathString)
+ .add("db", dbName).add("table", tableName)
+ .add("partition", Optional.absent().orNull()).toString();
+ Assert.assertEquals(actualString, simpleHiveSpec.toString());
+ }
+
+ @Test(priority=2)
+ public void testActivity(){
+ DropPartitionActivity dropPartitionActivity = new DropPartitionActivity(dbName, tableName,
+ new ArrayList<>(), new ArrayList<>());
+ DropTableActivity dropTableActivity = new DropTableActivity(dbName, tableName);
+
+ simpleHiveSpec.getPreActivities().add(dropPartitionActivity);
+ simpleHiveSpec.getPreActivities().add(dropTableActivity);
+
+ simpleHiveSpec.getPostActivities().add(dropPartitionActivity);
+
+ Assert.assertEquals(simpleHiveSpec.getPreActivities().size(), 2);
+ Assert.assertEquals(simpleHiveSpec.getPostActivities().size(), 1);
+ }
+
+ @Test(priority=3)
+ public void testPredicate(){
+ TableNotExistPredicate tableNotExistPredicate = new TableNotExistPredicate(dbName, tableName);
+ PartitionNotExistPredicate partitionNotExistPredicate = new PartitionNotExistPredicate(dbName, tableName,
+ new ArrayList<>(), new ArrayList<>());
+
+ simpleHiveSpec.getPredicates().add(tableNotExistPredicate);
+ simpleHiveSpec.getPredicates().add(partitionNotExistPredicate);
+
+ Assert.assertEquals(simpleHiveSpec.getPredicates().size(), 2);
+ }
+}
diff --git a/gobblin-hive-registration/src/test/resources/test-hive-table/snapshot1/.gitignore b/gobblin-hive-registration/src/test/resources/test-hive-table/snapshot1/.gitignore
new file mode 100644
index 0000000..c9eebd2
--- /dev/null
+++ b/gobblin-hive-registration/src/test/resources/test-hive-table/snapshot1/.gitignore
@@ -0,0 +1,5 @@
+# Ignore everything in this directory
+*
+# Except this file for keeping this dummy directory.
+# This dummy directory is for testing the hive snapshot registration.
+!.gitignore
\ No newline at end of file