[incubator-livy] branch master updated: [LIVY-707] Add audit log for SqlJobs from ThriftServer

[js...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jshao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-livy.git


The following commit(s) were added to refs/heads/master by this push:
     new 6261c57  [LIVY-707] Add audit log for SqlJobs from ThriftServer
6261c57 is described below

commit 6261c57be8df66d5f3fc3ccdaa15f8c4e1989d1d
Author: BoneAn <an...@oppo.com>
AuthorDate: Thu Nov 14 09:51:48 2019 +0800

    [LIVY-707] Add audit log for SqlJobs from ThriftServer
    
    ## What changes were proposed in this pull request?
    
    We should add audit logs in thriftServer for admin to easily to manage operations,
    
    ## How was this patch tested?
    
    An audit log example showed below,
    
    ```
    19/11/06 16:38:30 INFO ThriftServerAudit$: user: test ipAddress: 10.25.22.46 query: select count(*) from test1 beforeExecute: 1573029416951 afterExecute: 1573029510972 time spent: 94021
    ```
    
    Author: BoneAn <an...@oppo.com>
    
    Closes #255 from huianyi/LIVY-707.
---
 .../LivyExecuteStatementOperation.scala            |  6 ++++
 .../livy/thriftserver/ThriftServerAudit.scala      | 36 ++++++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyExecuteStatementOperation.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyExecuteStatementOperation.scala
index ebb8e1d..f7d6c16 100644
--- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyExecuteStatementOperation.scala
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyExecuteStatementOperation.scala
@@ -137,6 +137,8 @@ class LivyExecuteStatementOperation(
     }
     setState(OperationState.RUNNING)
 
+    val before = System.currentTimeMillis()
+
     try {
       rpcClient.executeSql(sessionHandle, statementId, statement).get()
     } catch {
@@ -147,6 +149,10 @@ class LivyExecuteStatementOperation(
         throw new HiveSQLException(e)
     }
     setState(OperationState.FINISHED)
+
+    val sessionInfo = sessionManager.getSessionInfo(sessionHandle)
+    val after = System.currentTimeMillis()
+    ThriftServerAudit.audit(sessionInfo.username, sessionInfo.ipAddress, statement, before, after)
   }
 
   def close(): Unit = {
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerAudit.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerAudit.scala
new file mode 100644
index 0000000..5bf7760
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerAudit.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver
+
+import org.apache.livy.Logging
+
+object ThriftServerAudit extends Logging {
+
+  def audit(
+      user: String,
+      ipAddress: String,
+      query: String,
+      startTime: Long,
+      endTime: Long): Unit = {
+    info(
+      s"user: $user ipAddress: $ipAddress query: ${query.replace('\n', ' ')} " +
+        s"start time: ${startTime} end time: ${endTime} " +
+        s"time spent: ${Math.round((endTime - startTime) / 1000)}s")
+  }
+
+}