You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2018/12/14 13:22:36 UTC
hive git commit: HIVE-21035: Race condition in
SparkUtilities#getSparkSession (Antal Sinkovits, reviewed by Adam Szita,
Denys Kuzmenko)
Repository: hive
Updated Branches:
refs/heads/master 7da8f3d36 -> 687aeef53
HIVE-21035: Race condition in SparkUtilities#getSparkSession (Antal Sinkovits, reviewed by Adam Szita, Denys Kuzmenko)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/687aeef5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/687aeef5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/687aeef5
Branch: refs/heads/master
Commit: 687aeef5347b9ee28f41aacb3085cf513e01afe1
Parents: 7da8f3d
Author: Antal Sinkovits <as...@cloudera.com>
Authored: Thu Dec 13 11:13:18 2018 +0100
Committer: Adam Szita <sz...@cloudera.com>
Committed: Fri Dec 14 14:13:28 2018 +0100
----------------------------------------------------------------------
.../hive/ql/exec/spark/SparkUtilities.java | 32 ++---
.../hive/ql/exec/spark/TestSparkUtilities.java | 117 +++++++++++++++++++
2 files changed, 135 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/687aeef5/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
index d384ed6..fafae31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -122,21 +122,25 @@ public class SparkUtilities {
public static SparkSession getSparkSession(HiveConf conf,
SparkSessionManager sparkSessionManager) throws HiveException {
- SparkSession sparkSession = SessionState.get().getSparkSession();
- HiveConf sessionConf = SessionState.get().getConf();
- // Spark configurations are updated close the existing session
- // In case of async queries or confOverlay is not empty,
- // sessionConf and conf are different objects
- if (sessionConf.getSparkConfigUpdated() || conf.getSparkConfigUpdated()) {
- sparkSessionManager.closeSession(sparkSession);
- sparkSession = null;
- conf.setSparkConfigUpdated(false);
- sessionConf.setSparkConfigUpdated(false);
+ SessionState sessionState = SessionState.get();
+ synchronized (sessionState) {
+ SparkSession sparkSession = sessionState.getSparkSession();
+ HiveConf sessionConf = sessionState.getConf();
+
+ // Spark configurations are updated close the existing session
+ // In case of async queries or confOverlay is not empty,
+ // sessionConf and conf are different objects
+ if (sessionConf.getSparkConfigUpdated() || conf.getSparkConfigUpdated()) {
+ sparkSessionManager.closeSession(sparkSession);
+ sparkSession = null;
+ conf.setSparkConfigUpdated(false);
+ sessionConf.setSparkConfigUpdated(false);
+ }
+ sparkSession = sparkSessionManager.getSession(sparkSession, conf, true);
+ sessionState.setSparkSession(sparkSession);
+ return sparkSession;
}
- sparkSession = sparkSessionManager.getSession(sparkSession, conf, true);
- SessionState.get().setSparkSession(sparkSession);
- return sparkSession;
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/687aeef5/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkUtilities.java
new file mode 100644
index 0000000..f797f30
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkUtilities.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.spark;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
+import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManager;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for the SparkUtilities class.
+ */
+public class TestSparkUtilities {
+
+ @Test
+ public void testGetSparkSessionUsingMultipleThreadsWithTheSameSession() throws HiveException, InterruptedException {
+
+ // The only real state required from SessionState
+ final AtomicReference<SparkSession> activeSparkSession = new AtomicReference<>();
+
+ // Mocks
+ HiveConf mockConf = mock(HiveConf.class);
+
+ SparkSessionManager mockSessionManager = mock(SparkSessionManager.class);
+ doAnswer(invocationOnMock -> {
+ SparkSession sparkSession = invocationOnMock.getArgumentAt(0, SparkSession.class);
+ if (sparkSession == null) {
+ return mock(SparkSession.class);
+ } else {
+ return sparkSession;
+ }
+ }).when(mockSessionManager).getSession(any(SparkSession.class), eq(mockConf), eq(true));
+
+ SessionState mockSessionState = mock(SessionState.class);
+ when(mockSessionState.getConf()).thenReturn(mockConf);
+ doAnswer(invocationOnMock -> {
+ activeSparkSession.set(invocationOnMock.getArgumentAt(0, SparkSession.class));
+ return null;
+ }).when(mockSessionState).setSparkSession(any(SparkSession.class));
+ doAnswer(invocationOnMock ->
+ activeSparkSession.get()
+ ).when(mockSessionState).getSparkSession();
+
+ // When
+ List<Callable<SparkSession>> callables = new ArrayList<>();
+ callables.add(new GetSparkSessionTester(mockConf, mockSessionManager, mockSessionState));
+ callables.add(new GetSparkSessionTester(mockConf, mockSessionManager, mockSessionState));
+ callables.add(new GetSparkSessionTester(mockConf, mockSessionManager, mockSessionState));
+
+ ExecutorService executorService = Executors.newFixedThreadPool(callables.size());
+ List<Future<SparkSession>> results = executorService.invokeAll(callables);
+
+ // Then
+ results.stream().map(f -> resolve(f)).forEach(ss -> assertEquals(ss, activeSparkSession.get()));
+
+ }
+
+ private SparkSession resolve(Future<SparkSession> future) {
+ try {
+ return future.get();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private class GetSparkSessionTester implements Callable<SparkSession> {
+
+ private HiveConf hiveConf;
+ private SparkSessionManager sparkSessionManager;
+ private SessionState sessionState;
+
+ GetSparkSessionTester(HiveConf hiveConf, SparkSessionManager sparkSessionManager,
+ SessionState sessionState) {
+ this.hiveConf = hiveConf;
+ this.sparkSessionManager = sparkSessionManager;
+ this.sessionState = sessionState;
+ }
+
+ @Override
+ public SparkSession call() throws Exception {
+ SessionState.setCurrentSessionState(sessionState);
+ return SparkUtilities.getSparkSession(hiveConf, sparkSessionManager);
+ }
+ }
+}