You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/11/03 22:49:48 UTC
[4/7] hive git commit: HIVE-17967 Move HiveMetaStore class
http://git-wip-us.apache.org/repos/asf/hive/blob/c2bbd5f4/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore2.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore2.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore2.java
deleted file mode 100644
index fa4e02a..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore2.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.metastore.messaging.EventMessage;
-import org.apache.hadoop.hive.metastore.model.MNotificationLog;
-import org.apache.hadoop.hive.metastore.model.MNotificationNextId;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.hive.metastore.TestOldSchema.dropAllStoreObjects;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-// Tests from TestObjectStore that can't be moved yet due to references to EventMessage. Once
-// EventMessage has been moved this should be recombined with TestObjectStore.
-
-public class TestObjectStore2 {
- private ObjectStore objectStore = null;
-
- public static class MockPartitionExpressionProxy implements PartitionExpressionProxy {
- @Override
- public String convertExprToFilter(byte[] expr) throws MetaException {
- return null;
- }
-
- @Override
- public boolean filterPartitionsByExpr(List<FieldSchema> partColumns,
- byte[] expr, String defaultPartitionName, List<String> partitionNames)
- throws MetaException {
- return false;
- }
-
- @Override
- public FileMetadataExprType getMetadataType(String inputFormat) {
- return null;
- }
-
- @Override
- public SearchArgument createSarg(byte[] expr) {
- return null;
- }
-
- @Override
- public FileFormatProxy getFileFormatProxy(FileMetadataExprType type) {
- return null;
- }
- }
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = MetastoreConf.newMetastoreConf();
- MetastoreConf.setVar(conf, MetastoreConf.ConfVars.EXPRESSION_PROXY_CLASS,
- MockPartitionExpressionProxy.class.getName());
-
- objectStore = new ObjectStore();
- objectStore.setConf(conf);
- dropAllStoreObjects(objectStore);
- }
-
- /**
- * Test notification operations
- */
- // TODO MS-SPLIT uncomment once we move EventMessage over
- @Test
- public void testNotificationOps() throws InterruptedException {
- final int NO_EVENT_ID = 0;
- final int FIRST_EVENT_ID = 1;
- final int SECOND_EVENT_ID = 2;
-
- NotificationEvent event =
- new NotificationEvent(0, 0, EventMessage.EventType.CREATE_DATABASE.toString(), "");
- NotificationEventResponse eventResponse;
- CurrentNotificationEventId eventId;
-
- // Verify that there is no notifications available yet
- eventId = objectStore.getCurrentNotificationEventId();
- assertEquals(NO_EVENT_ID, eventId.getEventId());
-
- // Verify that addNotificationEvent() updates the NotificationEvent with the new event ID
- objectStore.addNotificationEvent(event);
- assertEquals(FIRST_EVENT_ID, event.getEventId());
- objectStore.addNotificationEvent(event);
- assertEquals(SECOND_EVENT_ID, event.getEventId());
-
- // Verify that objectStore fetches the latest notification event ID
- eventId = objectStore.getCurrentNotificationEventId();
- assertEquals(SECOND_EVENT_ID, eventId.getEventId());
-
- // Verify that getNextNotification() returns all events
- eventResponse = objectStore.getNextNotification(new NotificationEventRequest());
- assertEquals(2, eventResponse.getEventsSize());
- assertEquals(FIRST_EVENT_ID, eventResponse.getEvents().get(0).getEventId());
- assertEquals(SECOND_EVENT_ID, eventResponse.getEvents().get(1).getEventId());
-
- // Verify that getNextNotification(last) returns events after a specified event
- eventResponse = objectStore.getNextNotification(new NotificationEventRequest(FIRST_EVENT_ID));
- assertEquals(1, eventResponse.getEventsSize());
- assertEquals(SECOND_EVENT_ID, eventResponse.getEvents().get(0).getEventId());
-
- // Verify that getNextNotification(last) returns zero events if there are no more notifications available
- eventResponse = objectStore.getNextNotification(new NotificationEventRequest(SECOND_EVENT_ID));
- assertEquals(0, eventResponse.getEventsSize());
-
- // Verify that cleanNotificationEvents() cleans up all old notifications
- Thread.sleep(1);
- objectStore.cleanNotificationEvents(1);
- eventResponse = objectStore.getNextNotification(new NotificationEventRequest());
- assertEquals(0, eventResponse.getEventsSize());
- }
-
- @Ignore(
- "This test is here to allow testing with other databases like mysql / postgres etc\n"
- + " with user changes to the code. This cannot be run on apache derby because of\n"
- + " https://db.apache.org/derby/docs/10.10/devguide/cdevconcepts842385.html"
- )
- @Test
- public void testConcurrentAddNotifications() throws ExecutionException, InterruptedException {
-
- final int NUM_THREADS = 10;
- CyclicBarrier cyclicBarrier = new CyclicBarrier(NUM_THREADS,
- () -> LoggerFactory.getLogger("test")
- .debug(NUM_THREADS + " threads going to add notification"));
-
- Configuration conf = MetastoreConf.newMetastoreConf();
- MetastoreConf.setVar(conf, MetastoreConf.ConfVars.EXPRESSION_PROXY_CLASS,
- MockPartitionExpressionProxy.class.getName());
- /*
- Below are the properties that need to be set based on what database this test is going to be run
- */
-
-// conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "com.mysql.jdbc.Driver");
-// conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,
-// "jdbc:mysql://localhost:3306/metastore_db");
-// conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, "");
-// conf.setVar(HiveConf.ConfVars.METASTOREPWD, "");
-
- /*
- we have to add this one manually as for tests the db is initialized via the metastoreDiretSQL
- and we don't run the schema creation sql that includes the an insert for notification_sequence
- which can be locked. the entry in notification_sequence happens via notification_event insertion.
- */
- objectStore.getPersistenceManager().newQuery(MNotificationLog.class, "eventType==''").execute();
- objectStore.getPersistenceManager().newQuery(MNotificationNextId.class, "nextEventId==-1").execute();
-
- objectStore.addNotificationEvent(
- new NotificationEvent(0, 0,
- EventMessage.EventType.CREATE_DATABASE.toString(),
- "CREATE DATABASE DB initial"));
-
- ExecutorService executorService = Executors.newFixedThreadPool(NUM_THREADS);
- for (int i = 0; i < NUM_THREADS; i++) {
- final int n = i;
-
- executorService.execute(
- () -> {
- ObjectStore store = new ObjectStore();
- store.setConf(conf);
-
- String eventType = EventMessage.EventType.CREATE_DATABASE.toString();
- NotificationEvent dbEvent =
- new NotificationEvent(0, 0, eventType,
- "CREATE DATABASE DB" + n);
- System.out.println("ADDING NOTIFICATION");
-
- try {
- cyclicBarrier.await();
- } catch (InterruptedException | BrokenBarrierException e) {
- throw new RuntimeException(e);
- }
- store.addNotificationEvent(dbEvent);
- System.out.println("FINISH NOTIFICATION");
- });
- }
- executorService.shutdown();
- assertTrue(executorService.awaitTermination(15, TimeUnit.SECONDS));
-
- // we have to setup this again as the underlying PMF keeps getting reinitialized with original
- // reference closed
- ObjectStore store = new ObjectStore();
- store.setConf(conf);
-
- NotificationEventResponse eventResponse = store.getNextNotification(
- new NotificationEventRequest());
- assertEquals(NUM_THREADS + 1, eventResponse.getEventsSize());
- long previousId = 0;
- for (NotificationEvent event : eventResponse.getEvents()) {
- assertTrue("previous:" + previousId + " current:" + event.getEventId(),
- previousId < event.getEventId());
- assertTrue(previousId + 1 == event.getEventId());
- previousId = event.getEventId();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/c2bbd5f4/standalone-metastore/pom.xml
----------------------------------------------------------------------
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index a2a34a5..eee6528 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -522,6 +522,7 @@
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr3-maven-plugin</artifactId>
+ <version>${antlr.version}</version>
<executions>
<execution>
<goals>
http://git-wip-us.apache.org/repos/asf/hive/blob/c2bbd5f4/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultPartitionExpressionProxy.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultPartitionExpressionProxy.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultPartitionExpressionProxy.java
new file mode 100644
index 0000000..ec543be
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultPartitionExpressionProxy.java
@@ -0,0 +1,57 @@
+package org.apache.hadoop.hive.metastore;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+
+import java.util.List;
+
+/**
+ * Default implementation of PartitionExpressionProxy. Eventually this should use the SARGs in
+ * Hive's storage-api. For now it just throws UnsupportedOperationException.
+ */
+public class DefaultPartitionExpressionProxy implements PartitionExpressionProxy {
+ @Override
+ public String convertExprToFilter(byte[] expr) throws MetaException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean filterPartitionsByExpr(List<FieldSchema> partColumns, byte[] expr, String
+ defaultPartitionName, List<String> partitionNames) throws MetaException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FileMetadataExprType getMetadataType(String inputFormat) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FileFormatProxy getFileFormatProxy(FileMetadataExprType type) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SearchArgument createSarg(byte[] expr) {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c2bbd5f4/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java
new file mode 100644
index 0000000..1dbfa42
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+import java.util.List;
+
+/**
+ * Default StorageSchemaReader. This just throws as the metastore currently doesn't know how to
+ * read schemas from storage.
+ */
+public class DefaultStorageSchemaReader implements StorageSchemaReader {
+ @Override
+ public List<FieldSchema> readSchema(Table tbl, EnvironmentContext envContext,
+ Configuration conf) throws MetaException {
+ throw new UnsupportedOperationException("Storage schema reading not supported");
+ }
+}