You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2018/12/14 13:22:36 UTC

hive git commit: HIVE-21035: Race condition in SparkUtilities#getSparkSession (Antal Sinkovits, reviewed by Adam Szita, Denys Kuzmenko)

Repository: hive
Updated Branches:
  refs/heads/master 7da8f3d36 -> 687aeef53


HIVE-21035: Race condition in SparkUtilities#getSparkSession (Antal Sinkovits, reviewed by Adam Szita, Denys Kuzmenko)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/687aeef5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/687aeef5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/687aeef5

Branch: refs/heads/master
Commit: 687aeef5347b9ee28f41aacb3085cf513e01afe1
Parents: 7da8f3d
Author: Antal Sinkovits <as...@cloudera.com>
Authored: Thu Dec 13 11:13:18 2018 +0100
Committer: Adam Szita <sz...@cloudera.com>
Committed: Fri Dec 14 14:13:28 2018 +0100

----------------------------------------------------------------------
 .../hive/ql/exec/spark/SparkUtilities.java      |  32 ++---
 .../hive/ql/exec/spark/TestSparkUtilities.java  | 117 +++++++++++++++++++
 2 files changed, 135 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/687aeef5/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
index d384ed6..fafae31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -122,21 +122,25 @@ public class SparkUtilities {
 
   public static SparkSession getSparkSession(HiveConf conf,
       SparkSessionManager sparkSessionManager) throws HiveException {
-    SparkSession sparkSession = SessionState.get().getSparkSession();
-    HiveConf sessionConf = SessionState.get().getConf();
 
-    // Spark configurations are updated close the existing session
-    // In case of async queries or confOverlay is not empty,
-    // sessionConf and conf are different objects
-    if (sessionConf.getSparkConfigUpdated() || conf.getSparkConfigUpdated()) {
-      sparkSessionManager.closeSession(sparkSession);
-      sparkSession = null;
-      conf.setSparkConfigUpdated(false);
-      sessionConf.setSparkConfigUpdated(false);
+    SessionState sessionState = SessionState.get();
+    synchronized (sessionState) {
+      SparkSession sparkSession = sessionState.getSparkSession();
+      HiveConf sessionConf = sessionState.getConf();
+
+      // Spark configurations are updated close the existing session
+      // In case of async queries or confOverlay is not empty,
+      // sessionConf and conf are different objects
+      if (sessionConf.getSparkConfigUpdated() || conf.getSparkConfigUpdated()) {
+        sparkSessionManager.closeSession(sparkSession);
+        sparkSession = null;
+        conf.setSparkConfigUpdated(false);
+        sessionConf.setSparkConfigUpdated(false);
+      }
+      sparkSession = sparkSessionManager.getSession(sparkSession, conf, true);
+      sessionState.setSparkSession(sparkSession);
+      return sparkSession;
     }
-    sparkSession = sparkSessionManager.getSession(sparkSession, conf, true);
-    SessionState.get().setSparkSession(sparkSession);
-    return sparkSession;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/687aeef5/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkUtilities.java
new file mode 100644
index 0000000..f797f30
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkUtilities.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.spark;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
+import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManager;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for the SparkUtilities class.
+ */
+public class TestSparkUtilities {
+
+  @Test
+  public void testGetSparkSessionUsingMultipleThreadsWithTheSameSession() throws HiveException, InterruptedException {
+
+    // The only real state required from SessionState
+    final AtomicReference<SparkSession> activeSparkSession = new AtomicReference<>();
+
+    // Mocks
+    HiveConf mockConf = mock(HiveConf.class);
+
+    SparkSessionManager mockSessionManager = mock(SparkSessionManager.class);
+    doAnswer(invocationOnMock -> {
+      SparkSession sparkSession = invocationOnMock.getArgumentAt(0, SparkSession.class);
+      if (sparkSession == null) {
+        return mock(SparkSession.class);
+      } else {
+        return sparkSession;
+      }
+    }).when(mockSessionManager).getSession(any(SparkSession.class), eq(mockConf), eq(true));
+
+    SessionState mockSessionState = mock(SessionState.class);
+    when(mockSessionState.getConf()).thenReturn(mockConf);
+    doAnswer(invocationOnMock -> {
+      activeSparkSession.set(invocationOnMock.getArgumentAt(0, SparkSession.class));
+      return null;
+    }).when(mockSessionState).setSparkSession(any(SparkSession.class));
+    doAnswer(invocationOnMock ->
+      activeSparkSession.get()
+    ).when(mockSessionState).getSparkSession();
+
+    // When
+    List<Callable<SparkSession>> callables = new ArrayList<>();
+    callables.add(new GetSparkSessionTester(mockConf, mockSessionManager, mockSessionState));
+    callables.add(new GetSparkSessionTester(mockConf, mockSessionManager, mockSessionState));
+    callables.add(new GetSparkSessionTester(mockConf, mockSessionManager, mockSessionState));
+
+    ExecutorService executorService = Executors.newFixedThreadPool(callables.size());
+    List<Future<SparkSession>> results = executorService.invokeAll(callables);
+
+    // Then
+    results.stream().map(f -> resolve(f)).forEach(ss -> assertEquals(ss, activeSparkSession.get()));
+
+  }
+
+  private SparkSession resolve(Future<SparkSession> future) {
+    try {
+      return future.get();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private class GetSparkSessionTester implements Callable<SparkSession> {
+
+    private HiveConf hiveConf;
+    private SparkSessionManager sparkSessionManager;
+    private SessionState sessionState;
+
+    GetSparkSessionTester(HiveConf hiveConf, SparkSessionManager sparkSessionManager,
+                                 SessionState sessionState) {
+      this.hiveConf = hiveConf;
+      this.sparkSessionManager = sparkSessionManager;
+      this.sessionState = sessionState;
+    }
+
+    @Override
+    public SparkSession call() throws Exception {
+      SessionState.setCurrentSessionState(sessionState);
+      return SparkUtilities.getSparkSession(hiveConf, sparkSessionManager);
+    }
+  }
+}