You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by va...@apache.org on 2018/11/30 23:57:34 UTC
[1/5] incubator-livy git commit: [LIVY-502] Remove dependency on
hive-exec
Repository: incubator-livy
Updated Branches:
refs/heads/master 39fa887cf -> 47d3ee6b6
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/serde/ThriftResultSet.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/serde/ThriftResultSet.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/serde/ThriftResultSet.scala
new file mode 100644
index 0000000..e555b34
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/serde/ThriftResultSet.scala
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.serde
+
+import java.nio.ByteBuffer
+import java.util
+
+import com.google.common.primitives.{Booleans, Bytes, Doubles, Ints, Longs, Shorts}
+import org.apache.hive.service.rpc.thrift.{TBinaryColumn, TBoolColumn, TByteColumn, TColumn, TDoubleColumn, TI16Column, TI32Column, TI64Column, TProtocolVersion, TRow, TRowSet, TStringColumn}
+import org.apache.hive.service.rpc.thrift.TProtocolVersion._
+
+import org.apache.livy.thriftserver.session.{ColumnBuffer, DataType, ResultSet}
+import org.apache.livy.thriftserver.types.Schema
+
+abstract class ThriftResultSet {
+ def toTRowSet: TRowSet
+ def addRow(row: Array[Any]): Unit
+ def setRowOffset(rowOffset: Long): Unit
+ def numRows: Int
+}
+
+object ThriftResultSet {
+ def apply(types: Array[DataType], protocolVersion: TProtocolVersion): ThriftResultSet = {
+ // Older version of Hive protocol require the result set to be returned in a row
+ // oriented way. We do not support those versions, so it is useless to implement it.
+ assert(protocolVersion.getValue >= HIVE_CLI_SERVICE_PROTOCOL_V6.getValue)
+ new ColumnOrientedResultSet(types.map(new ColumnBuffer(_)))
+ }
+
+ def apply(schema: Schema, protocolVersion: TProtocolVersion): ThriftResultSet = {
+ apply(schema.fields.map(_.fieldType.dataType), protocolVersion)
+ }
+
+ def apply(resultSet: ResultSet): ThriftResultSet = {
+ new ColumnOrientedResultSet(resultSet.getColumns)
+ }
+
+ def toTColumn(columnBuffer: ColumnBuffer): TColumn = {
+ val value = new TColumn
+ val nullsArray = columnBuffer.getNulls.toByteArray
+ val nullMasks = ByteBuffer.wrap(nullsArray)
+ columnBuffer.getType match {
+ case DataType.BOOLEAN =>
+ val bools = columnBuffer.getValues.asInstanceOf[Array[Boolean]]
+ value.setBoolVal(new TBoolColumn(Booleans.asList(bools: _*), nullMasks))
+ case DataType.BYTE =>
+ val bytes = columnBuffer.getValues.asInstanceOf[Array[Byte]]
+ value.setByteVal(new TByteColumn(Bytes.asList(bytes: _*), nullMasks))
+ case DataType.SHORT =>
+ val shorts = columnBuffer.getValues.asInstanceOf[Array[Short]]
+ value.setI16Val(new TI16Column(Shorts.asList(shorts: _*), nullMasks))
+ case DataType.INTEGER =>
+ val integers = columnBuffer.getValues.asInstanceOf[Array[Int]]
+ value.setI32Val(new TI32Column(Ints.asList(integers: _*), nullMasks))
+ case DataType.LONG =>
+ val longs = columnBuffer.getValues.asInstanceOf[Array[Long]]
+ value.setI64Val(new TI64Column(Longs.asList(longs: _*), nullMasks))
+ case DataType.FLOAT =>
+ val floats = columnBuffer.getValues.asInstanceOf[Array[Float]]
+ val doubles = new util.ArrayList[java.lang.Double](floats.length)
+ var i = 0
+ while (i < floats.length) {
+ doubles.add(floats(i).toString.toDouble)
+ i += 1
+ }
+ value.setDoubleVal(new TDoubleColumn(doubles, nullMasks))
+ case DataType.DOUBLE =>
+ val doubles = columnBuffer.getValues.asInstanceOf[Array[Double]]
+ value.setDoubleVal(new TDoubleColumn(Doubles.asList(doubles: _*), nullMasks))
+ case DataType.BINARY =>
+ val binaries = columnBuffer.getValues.asInstanceOf[util.List[ByteBuffer]]
+ value.setBinaryVal(new TBinaryColumn(binaries, nullMasks))
+ case _ =>
+ val strings = columnBuffer.getValues.asInstanceOf[util.List[String]]
+ value.setStringVal(new TStringColumn(strings, nullMasks))
+ }
+ value
+ }
+}
+
+/**
+ * [[ThriftResultSet]] implementation which uses columns to store data.
+ */
+class ColumnOrientedResultSet(
+ private val columns: Array[ColumnBuffer]) extends ThriftResultSet {
+
+ private var rowOffset: Long = _
+
+ def addRow(fields: Array[Any]): Unit = {
+ var i = 0
+ while (i < fields.length) {
+ val field = fields(i)
+ columns(i).add(field)
+ i += 1
+ }
+ }
+
+ override def toTRowSet: TRowSet = {
+ val tRowSet = new TRowSet(rowOffset, new util.ArrayList[TRow])
+ columns.foreach { c =>
+ tRowSet.addToColumns(ThriftResultSet.toTColumn(c))
+ }
+ tRowSet
+ }
+
+ override def setRowOffset(rowOffset: Long): Unit = this.rowOffset = rowOffset
+
+ override def numRows: Int = columns.headOption.map(_.size).getOrElse(0)
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/DataTypeUtils.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/DataTypeUtils.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/DataTypeUtils.scala
index f61dd30..8220c27 100644
--- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/DataTypeUtils.scala
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/DataTypeUtils.scala
@@ -17,17 +17,10 @@
package org.apache.livy.thriftserver.types
-import scala.collection.JavaConverters._
-
-import org.apache.hadoop.hive.metastore.api.FieldSchema
-import org.apache.hadoop.hive.serde2.thrift.Type
-import org.apache.hive.service.cli.TableSchema
import org.json4s.{DefaultFormats, JValue}
import org.json4s.JsonAST.{JObject, JString}
import org.json4s.jackson.JsonMethods.parse
-import org.apache.livy.thriftserver.session.DataType
-
/**
* Utility class for converting and representing Spark and Hive data types.
*/
@@ -35,71 +28,45 @@ object DataTypeUtils {
// Used for JSON conversion
private implicit val formats = DefaultFormats
- /**
- * Returns the Hive [[Type]] used in the thrift communications for the given Livy type.
- */
- def toHiveThriftType(ltype: DataType): Type = {
- ltype match {
- case DataType.BOOLEAN => Type.BOOLEAN_TYPE
- case DataType.BYTE => Type.TINYINT_TYPE
- case DataType.SHORT => Type.SMALLINT_TYPE
- case DataType.INTEGER => Type.INT_TYPE
- case DataType.LONG => Type.BIGINT_TYPE
- case DataType.FLOAT => Type.FLOAT_TYPE
- case DataType.DOUBLE => Type.DOUBLE_TYPE
- case DataType.BINARY => Type.BINARY_TYPE
- case _ => Type.STRING_TYPE
+ private def toFieldType(jValue: JValue): FieldType = {
+ jValue match {
+ case JString(t) => BasicDataType(t)
+ case o: JObject => complexToDataType(o)
+ case _ => throw new IllegalArgumentException(
+ s"Spark type was neither a string nor a object. It was: $jValue.")
+ }
+ }
+
+ private def complexToDataType(sparkType: JObject): FieldType = {
+ (sparkType \ "type").extract[String] match {
+ case "array" => ArrayType(toFieldType(sparkType \ "elementType"))
+ case "struct" =>
+ val fields = (sparkType \ "fields").children.map { f =>
+ // TODO: get comment from metadata
+ Field((f \ "name").extract[String], toFieldType(f \ "type"), "")
+ }
+ StructType(fields.toArray)
+ case "map" =>
+ MapType(toFieldType(sparkType \ "keyType"), toFieldType(sparkType \ "valueType"))
+ case "udt" => toFieldType(sparkType \ "sqlType")
}
}
/**
* Converts a JSON representing the Spark schema (the one returned by `df.schema.json`) into
- * a Hive [[TableSchema]] instance.
+ * a [[Schema]] instance.
*
* @param sparkJson a [[String]] containing the JSON representation of a Spark Dataframe schema
- * @return a [[TableSchema]] representing the schema provided as input
+ * @return a [[Schema]] representing the schema provided as input
*/
- def toHiveTableSchema(sparkJson: String): TableSchema = {
+ def schemaFromSparkJson(sparkJson: String): Schema = {
val schema = parse(sparkJson) \ "fields"
val fields = schema.children.map { field =>
val name = (field \ "name").extract[String]
- val hiveType = toHive(field \ "type")
- new FieldSchema(name, hiveType, "")
- }
- new TableSchema(fields.asJava)
- }
-
- private def toHive(jValue: JValue): String = {
- jValue match {
- case JString(t) => primitiveToHive(t)
- case o: JObject => complexToHive(o)
- case _ => throw new IllegalArgumentException(
- s"Spark type was neither a string nor a object. It was: $jValue.")
- }
- }
-
- private def primitiveToHive(sparkType: String): String = {
- sparkType match {
- case "integer" => "int"
- case "long" => "bigint"
- case "short" => "smallint"
- case "byte" => "tinyint"
- case "null" => "void"
- // boolean, string, float, double, decimal, date, timestamp are the same
- case other => other
- }
- }
-
- private def complexToHive(sparkType: JObject): String = {
- (sparkType \ "type").extract[String] match {
- case "array" => s"array<${toHive(sparkType \ "elementType")}>"
- case "struct" =>
- val fields = (sparkType \ "fields").children.map { f =>
- s"${(f \ "name").extract[String]}:${toHive(f \ "type")}"
- }
- s"struct<${fields.mkString(",")}>"
- case "map" => s"map<${toHive(sparkType \ "keyType")}, ${toHive(sparkType \ "valueType")}>"
- case "udt" => toHive(sparkType \ "sqlType")
+ val hiveType = toFieldType(field \ "type")
+ // TODO: retrieve comment from metadata
+ Field(name, hiveType, "")
}
+ Schema(fields.toArray)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala
new file mode 100644
index 0000000..6e06474
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.types
+
+import org.apache.hive.service.rpc.thrift.{TColumnDesc, TPrimitiveTypeEntry, TTableSchema, TTypeDesc, TTypeEntry, TTypeId}
+
+import org.apache.livy.thriftserver.session.DataType
+
+private[thriftserver] trait FieldType {
+ def name: String
+ def dataType: DataType
+}
+
+sealed trait ComplexFieldType extends FieldType {
+ override val dataType = DataType.STRING
+}
+
+case class BasicDataType(name: String) extends FieldType {
+
+ override val dataType: DataType = name match {
+ case "boolean" => DataType.BOOLEAN
+ case "byte" => DataType.BYTE
+ case "short" => DataType.SHORT
+ case "integer" => DataType.INTEGER
+ case "long" => DataType.LONG
+ case "float" => DataType.FLOAT
+ case "double" => DataType.DOUBLE
+ case "binary" => DataType.BINARY
+ case _ => DataType.STRING
+ }
+}
+
+case class StructType(fields: Array[Field]) extends ComplexFieldType {
+ override val name = "struct"
+}
+
+case class ArrayType(elementsType: FieldType) extends ComplexFieldType {
+ val name = "array"
+}
+
+case class MapType(keyType: FieldType, valueType: FieldType) extends ComplexFieldType {
+ val name = "map"
+}
+
+case class Field(name: String, fieldType: FieldType, comment: String)
+
+class Schema(val fields: Array[Field]) {
+
+ def toTTableSchema: TTableSchema = {
+ val tTableSchema = new TTableSchema()
+ fields.zipWithIndex.foreach { case (f, idx) =>
+ tTableSchema.addToColumns(Schema.columnDescriptor(f, idx))
+ }
+ tTableSchema
+ }
+}
+
+object Schema {
+ def apply(fields: Field*): Schema = new Schema(fields.toArray)
+
+ def apply(fields: Array[Field]): Schema = new Schema(fields)
+
+ def apply(names: Array[String], types: Array[FieldType]): Schema = {
+ assert(names.length == types.length)
+ val fields = names.zip(types).map { case (n, dt) => Field(n, dt, "") }
+ apply(fields)
+ }
+
+ private def columnDescriptor(field: Field, index: Int): TColumnDesc = {
+ val tColumnDesc = new TColumnDesc
+ tColumnDesc.setColumnName(field.name)
+ tColumnDesc.setComment(field.comment)
+ tColumnDesc.setTypeDesc(toTTypeDesc(field.fieldType.dataType))
+ tColumnDesc.setPosition(index)
+ tColumnDesc
+ }
+
+ private def toTTypeDesc(dt: DataType): TTypeDesc = {
+ val typeId = dt match {
+ case DataType.BOOLEAN => TTypeId.BOOLEAN_TYPE
+ case DataType.BYTE => TTypeId.TINYINT_TYPE
+ case DataType.SHORT => TTypeId.SMALLINT_TYPE
+ case DataType.INTEGER => TTypeId.INT_TYPE
+ case DataType.LONG => TTypeId.BIGINT_TYPE
+ case DataType.FLOAT => TTypeId.FLOAT_TYPE
+ case DataType.DOUBLE => TTypeId.DOUBLE_TYPE
+ case DataType.BINARY => TTypeId.BINARY_TYPE
+ case _ => TTypeId.STRING_TYPE
+ }
+ val primitiveEntry = new TPrimitiveTypeEntry(typeId)
+ val entry = TTypeEntry.primitiveEntry(primitiveEntry)
+ val desc = new TTypeDesc
+ desc.addToTypes(entry)
+ desc
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ui/ThriftJsonServlet.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ui/ThriftJsonServlet.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ui/ThriftJsonServlet.scala
index a7db3e9..ad1b5dc 100644
--- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ui/ThriftJsonServlet.scala
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ui/ThriftJsonServlet.scala
@@ -35,7 +35,7 @@ class ThriftJsonServlet(val basePath: String) extends JsonServlet {
get("/sessions") {
val thriftSessions = LivyThriftServer.getInstance.map { server =>
- val sessionManager = server.getSessionManager()
+ val sessionManager = server.getSessionManager
sessionManager.getSessions.map { sessionHandle =>
val info = sessionManager.getSessionInfo(sessionHandle)
SessionInfo(sessionHandle.getSessionId.toString,
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerBaseTest.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerBaseTest.scala b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerBaseTest.scala
index f1fb247..9a7823d 100644
--- a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerBaseTest.scala
+++ b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerBaseTest.scala
@@ -19,9 +19,7 @@ package org.apache.livy.thriftserver
import java.sql.{Connection, DriverManager, Statement}
-import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hive.jdbc.HiveDriver
-import org.apache.hive.service.Service.STATE
import org.scalatest.{BeforeAndAfterAll, FunSuite}
import org.apache.livy.LivyConf
@@ -56,13 +54,8 @@ abstract class ThriftServerBaseTest extends FunSuite with BeforeAndAfterAll {
override def beforeAll(): Unit = {
Class.forName(classOf[HiveDriver].getCanonicalName)
- livyConf.set(s"livy.${HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE}", mode.toString)
- val portConfKey = if (mode == ServerMode.http) {
- s"livy.${HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT}"
- } else {
- s"livy.${HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT}"
- }
- livyConf.set(portConfKey, port.toString)
+ livyConf.set(LivyConf.THRIFT_TRANSPORT_MODE, mode.toString)
+ livyConf.set(LivyConf.THRIFT_SERVER_PORT, port)
// Set formatted Spark and Scala version into livy configuration, this will be used by
// session creation.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
index 6eea6f3..3099436 100644
--- a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
+++ b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
@@ -77,7 +77,7 @@ class BinaryThriftServerSuite extends ThriftServerBaseTest with CommonThriftTest
test("Reuse existing session") {
withJdbcConnection { _ =>
- val sessionManager = LivyThriftServer.getInstance.get.getSessionManager()
+ val sessionManager = LivyThriftServer.getInstance.get.getSessionManager
val sessionHandle = sessionManager.getSessions.head
// Blocks until the session is ready
val session = sessionManager.getLivySession(sessionHandle)
@@ -106,7 +106,7 @@ class BinaryThriftServerSuite extends ThriftServerBaseTest with CommonThriftTest
val s1 = c.createStatement()
s1.execute(s"create database $db")
s1.close()
- val sessionManager = LivyThriftServer.getInstance.get.getSessionManager()
+ val sessionManager = LivyThriftServer.getInstance.get.getSessionManager
val sessionHandle = sessionManager.getSessions.head
// Blocks until the session is ready
val session = sessionManager.getLivySession(sessionHandle)
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/RegisterSessionJob.java
----------------------------------------------------------------------
diff --git a/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/RegisterSessionJob.java b/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/RegisterSessionJob.java
index 6a6bec3..03bad5d 100644
--- a/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/RegisterSessionJob.java
+++ b/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/RegisterSessionJob.java
@@ -25,7 +25,6 @@ import org.apache.livy.JobContext;
* job context.
*/
public class RegisterSessionJob implements Job<Boolean> {
-
private final String sessionId;
public RegisterSessionJob() {
@@ -41,5 +40,4 @@ public class RegisterSessionJob implements Job<Boolean> {
ThriftSessionState.create(ctx, sessionId);
return true;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/StatementState.java
----------------------------------------------------------------------
diff --git a/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/StatementState.java b/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/StatementState.java
index abbe866..5238845 100644
--- a/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/StatementState.java
+++ b/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/StatementState.java
@@ -22,8 +22,6 @@ import java.util.Iterator;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.types.StructType;
-import org.apache.livy.JobContext;
-
/**
* State related to one user statement.
*/
[4/5] incubator-livy git commit: [LIVY-502] Remove dependency on
hive-exec
Posted by va...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
deleted file mode 100644
index e16c6de..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ /dev/null
@@ -1,871 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import org.apache.hive.service.rpc.thrift.TSetClientInfoReq;
-import org.apache.hive.service.rpc.thrift.TSetClientInfoResp;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import javax.security.auth.login.LoginException;
-import org.apache.hadoop.hive.common.ServerUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.AbstractService;
-import org.apache.hive.service.ServiceException;
-import org.apache.hive.service.ServiceUtils;
-import org.apache.hive.service.auth.HiveAuthConstants;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.auth.TSetIpAddressProcessor;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.FetchType;
-import org.apache.hive.service.cli.GetInfoType;
-import org.apache.hive.service.cli.GetInfoValue;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.ICLIService;
-import org.apache.hive.service.cli.JobProgressUpdate;
-import org.apache.hive.service.cli.OperationHandle;
-import org.apache.hive.service.cli.OperationStatus;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.ProgressMonitorStatusMapper;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.rpc.thrift.TCLIService;
-import org.apache.hive.service.rpc.thrift.TCancelDelegationTokenReq;
-import org.apache.hive.service.rpc.thrift.TCancelDelegationTokenResp;
-import org.apache.hive.service.rpc.thrift.TCancelOperationReq;
-import org.apache.hive.service.rpc.thrift.TCancelOperationResp;
-import org.apache.hive.service.rpc.thrift.TCloseOperationReq;
-import org.apache.hive.service.rpc.thrift.TCloseOperationResp;
-import org.apache.hive.service.rpc.thrift.TCloseSessionReq;
-import org.apache.hive.service.rpc.thrift.TCloseSessionResp;
-import org.apache.hive.service.rpc.thrift.TExecuteStatementReq;
-import org.apache.hive.service.rpc.thrift.TExecuteStatementResp;
-import org.apache.hive.service.rpc.thrift.TFetchResultsReq;
-import org.apache.hive.service.rpc.thrift.TFetchResultsResp;
-import org.apache.hive.service.rpc.thrift.TGetCatalogsReq;
-import org.apache.hive.service.rpc.thrift.TGetCatalogsResp;
-import org.apache.hive.service.rpc.thrift.TGetColumnsReq;
-import org.apache.hive.service.rpc.thrift.TGetColumnsResp;
-import org.apache.hive.service.rpc.thrift.TGetCrossReferenceReq;
-import org.apache.hive.service.rpc.thrift.TGetCrossReferenceResp;
-import org.apache.hive.service.rpc.thrift.TGetDelegationTokenReq;
-import org.apache.hive.service.rpc.thrift.TGetDelegationTokenResp;
-import org.apache.hive.service.rpc.thrift.TGetFunctionsReq;
-import org.apache.hive.service.rpc.thrift.TGetFunctionsResp;
-import org.apache.hive.service.rpc.thrift.TGetInfoReq;
-import org.apache.hive.service.rpc.thrift.TGetInfoResp;
-import org.apache.hive.service.rpc.thrift.TGetOperationStatusReq;
-import org.apache.hive.service.rpc.thrift.TGetOperationStatusResp;
-import org.apache.hive.service.rpc.thrift.TGetPrimaryKeysReq;
-import org.apache.hive.service.rpc.thrift.TGetPrimaryKeysResp;
-import org.apache.hive.service.rpc.thrift.TGetQueryIdReq;
-import org.apache.hive.service.rpc.thrift.TGetQueryIdResp;
-import org.apache.hive.service.rpc.thrift.TGetResultSetMetadataReq;
-import org.apache.hive.service.rpc.thrift.TGetResultSetMetadataResp;
-import org.apache.hive.service.rpc.thrift.TGetSchemasReq;
-import org.apache.hive.service.rpc.thrift.TGetSchemasResp;
-import org.apache.hive.service.rpc.thrift.TGetTableTypesReq;
-import org.apache.hive.service.rpc.thrift.TGetTableTypesResp;
-import org.apache.hive.service.rpc.thrift.TGetTablesReq;
-import org.apache.hive.service.rpc.thrift.TGetTablesResp;
-import org.apache.hive.service.rpc.thrift.TGetTypeInfoReq;
-import org.apache.hive.service.rpc.thrift.TGetTypeInfoResp;
-import org.apache.hive.service.rpc.thrift.TJobExecutionStatus;
-import org.apache.hive.service.rpc.thrift.TOpenSessionReq;
-import org.apache.hive.service.rpc.thrift.TOpenSessionResp;
-import org.apache.hive.service.rpc.thrift.TProgressUpdateResp;
-import org.apache.hive.service.rpc.thrift.TProtocolVersion;
-import org.apache.hive.service.rpc.thrift.TRenewDelegationTokenReq;
-import org.apache.hive.service.rpc.thrift.TRenewDelegationTokenResp;
-import org.apache.hive.service.rpc.thrift.TStatus;
-import org.apache.hive.service.rpc.thrift.TStatusCode;
-import org.apache.hive.service.server.HiveServer2;
-import org.apache.thrift.TException;
-import org.apache.thrift.server.ServerContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.livy.thriftserver.LivyCLIService;
-import org.apache.livy.thriftserver.SessionInfo;
-
-
-/**
- * ThriftCLIService.
- *
- */
-public abstract class ThriftCLIService extends AbstractService implements TCLIService.Iface, Runnable {
-
- public static final Logger LOG = LoggerFactory.getLogger(ThriftCLIService.class.getName());
-
- protected ICLIService cliService;
- private static final TStatus OK_STATUS = new TStatus(TStatusCode.SUCCESS_STATUS);
- protected static HiveAuthFactory hiveAuthFactory;
-
- protected int portNum;
- protected InetAddress serverIPAddress;
- protected String hiveHost;
- private boolean isStarted = false;
- protected boolean isEmbedded = false;
-
- protected HiveConf hiveConf;
-
- protected int minWorkerThreads;
- protected int maxWorkerThreads;
- protected long workerKeepAliveTime;
- private Thread serverThread;
-
- protected ThreadLocal<ServerContext> currentServerContext;
-
- static class ThriftCLIServerContext implements ServerContext {
- private SessionHandle sessionHandle = null;
-
- public void setSessionHandle(SessionHandle sessionHandle) {
- this.sessionHandle = sessionHandle;
- }
-
- public SessionHandle getSessionHandle() {
- return sessionHandle;
- }
- }
-
- public ThriftCLIService(ICLIService service, String serviceName) {
- super(serviceName);
- this.cliService = service;
- currentServerContext = new ThreadLocal<ServerContext>();
- }
-
- @Override
- public synchronized void init(HiveConf hiveConf) {
- this.hiveConf = hiveConf;
-
- String hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST");
- if (hiveHost == null) {
- hiveHost = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST);
- }
- try {
- serverIPAddress = ServerUtils.getHostAddress(hiveHost);
- } catch (UnknownHostException e) {
- throw new ServiceException(e);
- }
-
- // Initialize common server configs needed in both binary & http modes
- String portString;
- // HTTP mode
- if (HiveServer2.isHTTPTransportMode(hiveConf)) {
- workerKeepAliveTime =
- hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME,
- TimeUnit.SECONDS);
- portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT");
- if (portString != null) {
- portNum = Integer.parseInt(portString);
- } else {
- portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT);
- }
- }
- // Binary mode
- else {
- workerKeepAliveTime =
- hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS);
- portString = System.getenv("HIVE_SERVER2_THRIFT_PORT");
- if (portString != null) {
- portNum = Integer.parseInt(portString);
- } else {
- portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT);
- }
- }
- minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS);
- maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS);
- super.init(hiveConf);
- }
-
- protected abstract void initServer();
-
- @Override
- public synchronized void start() {
- super.start();
- if (!isStarted && !isEmbedded) {
- initServer();
- serverThread = new Thread(this);
- serverThread.setName("Thrift Server");
- serverThread.start();
- isStarted = true;
- }
- }
-
- protected abstract void stopServer();
-
- @Override
- public synchronized void stop() {
- if (isStarted && !isEmbedded) {
- if (serverThread != null) {
- serverThread.interrupt();
- serverThread = null;
- }
- stopServer();
- isStarted = false;
- }
- super.stop();
- }
-
- public int getPortNumber() {
- return portNum;
- }
-
- public InetAddress getServerIPAddress() {
- return serverIPAddress;
- }
-
- @Override
- public TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req)
- throws TException {
- TGetDelegationTokenResp resp = new TGetDelegationTokenResp();
-
- if (hiveAuthFactory == null || !hiveAuthFactory.isSASLKerberosUser()) {
- resp.setStatus(unsecureTokenErrorStatus());
- } else {
- try {
- String token = cliService.getDelegationToken(
- new SessionHandle(req.getSessionHandle()),
- hiveAuthFactory, req.getOwner(), req.getRenewer());
- resp.setDelegationToken(token);
- resp.setStatus(OK_STATUS);
- } catch (HiveSQLException e) {
- LOG.error("Error obtaining delegation token", e);
- TStatus tokenErrorStatus = HiveSQLException.toTStatus(e);
- tokenErrorStatus.setSqlState("42000");
- resp.setStatus(tokenErrorStatus);
- }
- }
- return resp;
- }
-
- @Override
- public TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req)
- throws TException {
- TCancelDelegationTokenResp resp = new TCancelDelegationTokenResp();
-
- if (hiveAuthFactory == null || !hiveAuthFactory.isSASLKerberosUser()) {
- resp.setStatus(unsecureTokenErrorStatus());
- } else {
- try {
- cliService.cancelDelegationToken(new SessionHandle(req.getSessionHandle()),
- hiveAuthFactory, req.getDelegationToken());
- resp.setStatus(OK_STATUS);
- } catch (HiveSQLException e) {
- LOG.error("Error canceling delegation token", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- }
- return resp;
- }
-
- @Override
- public TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req)
- throws TException {
- TRenewDelegationTokenResp resp = new TRenewDelegationTokenResp();
- if (hiveAuthFactory == null || !hiveAuthFactory.isSASLKerberosUser()) {
- resp.setStatus(unsecureTokenErrorStatus());
- } else {
- try {
- cliService.renewDelegationToken(new SessionHandle(req.getSessionHandle()),
- hiveAuthFactory, req.getDelegationToken());
- resp.setStatus(OK_STATUS);
- } catch (HiveSQLException e) {
- LOG.error("Error obtaining renewing token", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- }
- return resp;
- }
-
- private TStatus unsecureTokenErrorStatus() {
- TStatus errorStatus = new TStatus(TStatusCode.ERROR_STATUS);
- errorStatus.setErrorMessage("Delegation token only supported over remote " +
- "client with kerberos authentication");
- return errorStatus;
- }
-
- @Override
- public TOpenSessionResp OpenSession(TOpenSessionReq req) throws TException {
- LOG.info("Client protocol version: " + req.getClient_protocol());
- TOpenSessionResp resp = new TOpenSessionResp();
- try {
- SessionHandle sessionHandle = getSessionHandle(req, resp);
- resp.setSessionHandle(sessionHandle.toTSessionHandle());
- Map<String, String> configurationMap = new HashMap<String, String>();
- // Set the updated fetch size from the server into the configuration map for the client
- String defaultFetchSize = Integer.toString(
- hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE));
- configurationMap.put(
- HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.varname,
- defaultFetchSize);
- resp.setConfiguration(configurationMap);
- resp.setStatus(OK_STATUS);
- ThriftCLIServerContext context =
- (ThriftCLIServerContext)currentServerContext.get();
- if (context != null) {
- context.setSessionHandle(sessionHandle);
- }
- } catch (Exception e) {
- LOG.warn("Error opening session: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TSetClientInfoResp SetClientInfo(TSetClientInfoReq req) throws TException {
- // TODO: We don't do anything for now, just log this for debugging.
- // We may be able to make use of this later, e.g. for workload management.
- TSetClientInfoResp resp = null;
- if (req.isSetConfiguration()) {
- StringBuilder sb = null;
- SessionHandle sh = null;
- for (Map.Entry<String, String> e : req.getConfiguration().entrySet()) {
- if (sb == null) {
- sh = new SessionHandle(req.getSessionHandle());
- sb = new StringBuilder("Client information for ").append(sh).append(": ");
- } else {
- sb.append(", ");
- }
- sb.append(e.getKey()).append(" = ").append(e.getValue());
- if ("ApplicationName".equals(e.getKey())) {
- try {
- cliService.setApplicationName(sh, e.getValue());
- } catch (Exception ex) {
- LOG.warn("Error setting application name", ex);
- resp = new TSetClientInfoResp(HiveSQLException.toTStatus(ex));
- }
- }
- }
- if (sb != null) {
- LOG.info("{}", sb);
- }
- }
- return resp == null ? new TSetClientInfoResp(OK_STATUS) : resp;
- }
-
- private String getIpAddress() {
- String clientIpAddress;
- // Http transport mode.
- // We set the thread local ip address, in ThriftHttpServlet.
- if (hiveConf.getVar(
- ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
- clientIpAddress = SessionInfo.getIpAddress();
- }
- else {
- if (hiveAuthFactory != null && hiveAuthFactory.isSASLWithKerberizedHadoop()) {
- clientIpAddress = hiveAuthFactory.getIpAddress();
- }
- // NOSASL
- else {
- clientIpAddress = TSetIpAddressProcessor.getUserIpAddress();
- }
- }
- LOG.debug("Client's IP Address: " + clientIpAddress);
- return clientIpAddress;
- }
-
- /**
- * Returns the effective username.
- * 1. If hive.server2.allow.user.substitution = false: the username of the connecting user
- * 2. If hive.server2.allow.user.substitution = true: the username of the end user,
- * that the connecting user is trying to proxy for.
- * This includes a check whether the connecting user is allowed to proxy for the end user.
- * @param req
- * @return
- * @throws HiveSQLException
- */
- private String getUserName(TOpenSessionReq req) throws HiveSQLException, IOException {
- String userName = null;
-
- if (hiveAuthFactory != null && hiveAuthFactory.isSASLWithKerberizedHadoop()) {
- userName = hiveAuthFactory.getRemoteUser();
- }
- // NOSASL
- if (userName == null) {
- userName = TSetIpAddressProcessor.getUserName();
- }
- // Http transport mode.
- // We set the thread local username, in ThriftHttpServlet.
- if (hiveConf.getVar(
- ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
- userName = SessionInfo.getUserName();
- }
- if (userName == null) {
- userName = req.getUsername();
- }
-
- userName = getShortName(userName);
- String effectiveClientUser = getProxyUser(userName, req.getConfiguration(), getIpAddress());
- LOG.debug("Client's username: " + effectiveClientUser);
- return effectiveClientUser;
- }
-
- private String getShortName(String userName) throws IOException {
- String ret = null;
-
- if (userName != null) {
- if (hiveAuthFactory != null && hiveAuthFactory.isSASLKerberosUser()) {
- // KerberosName.getShorName can only be used for kerberos user, but not for the user
- // logged in via other authentications such as LDAP
- KerberosNameShim fullKerberosName = ShimLoader.getHadoopShims().getKerberosNameShim(userName);
- ret = fullKerberosName.getShortName();
- } else {
- int indexOfDomainMatch = ServiceUtils.indexOfDomainMatch(userName);
- ret = (indexOfDomainMatch <= 0) ? userName : userName.substring(0, indexOfDomainMatch);
- }
- }
-
- return ret;
- }
-
- /**
- * Create a session handle
- * @param req
- * @param res
- * @return
- * @throws HiveSQLException
- * @throws LoginException
- * @throws IOException
- */
- SessionHandle getSessionHandle(TOpenSessionReq req, TOpenSessionResp res)
- throws HiveSQLException, LoginException, IOException {
- String userName = getUserName(req);
- String ipAddress = getIpAddress();
- TProtocolVersion protocol = getMinVersion(LivyCLIService.SERVER_VERSION(),
- req.getClient_protocol());
- SessionHandle sessionHandle;
- if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) &&
- (userName != null)) {
- sessionHandle = ((LivyCLIService) cliService).openSessionWithImpersonation(protocol,
- userName, req.getPassword(), ipAddress, req.getConfiguration(), null);
- } else {
- sessionHandle = ((LivyCLIService) cliService).openSession(protocol, userName,
- req.getPassword(), ipAddress, req.getConfiguration());
- }
- res.setServerProtocolVersion(protocol);
- return sessionHandle;
- }
-
- private double getProgressedPercentage(OperationHandle opHandle) throws HiveSQLException {
- checkArgument(OperationType.EXECUTE_STATEMENT.equals(opHandle.getOperationType()));
- return 0.0;
- }
-
- private TProtocolVersion getMinVersion(TProtocolVersion... versions) {
- TProtocolVersion[] values = TProtocolVersion.values();
- int current = values[values.length - 1].getValue();
- for (TProtocolVersion version : versions) {
- if (current > version.getValue()) {
- current = version.getValue();
- }
- }
- for (TProtocolVersion version : values) {
- if (version.getValue() == current) {
- return version;
- }
- }
- throw new IllegalArgumentException("never");
- }
-
- @Override
- public TCloseSessionResp CloseSession(TCloseSessionReq req) throws TException {
- TCloseSessionResp resp = new TCloseSessionResp();
- try {
- SessionHandle sessionHandle = new SessionHandle(req.getSessionHandle());
- cliService.closeSession(sessionHandle);
- resp.setStatus(OK_STATUS);
- ThriftCLIServerContext context =
- (ThriftCLIServerContext)currentServerContext.get();
- if (context != null) {
- context.setSessionHandle(null);
- }
- } catch (Exception e) {
- LOG.warn("Error closing session: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetInfoResp GetInfo(TGetInfoReq req) throws TException {
- TGetInfoResp resp = new TGetInfoResp();
- try {
- GetInfoValue getInfoValue =
- cliService.getInfo(new SessionHandle(req.getSessionHandle()),
- GetInfoType.getGetInfoType(req.getInfoType()));
- resp.setInfoValue(getInfoValue.toTGetInfoValue());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting info: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TExecuteStatementResp ExecuteStatement(TExecuteStatementReq req) throws TException {
- TExecuteStatementResp resp = new TExecuteStatementResp();
- try {
- SessionHandle sessionHandle = new SessionHandle(req.getSessionHandle());
- String statement = req.getStatement();
- Map<String, String> confOverlay = req.getConfOverlay();
- Boolean runAsync = req.isRunAsync();
- long queryTimeout = req.getQueryTimeout();
- OperationHandle operationHandle =
- runAsync ? cliService.executeStatementAsync(sessionHandle, statement, confOverlay,
- queryTimeout) : cliService.executeStatement(sessionHandle, statement, confOverlay,
- queryTimeout);
- resp.setOperationHandle(operationHandle.toTOperationHandle());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- // Note: it's rather important that this (and other methods) catch Exception, not Throwable;
- // in combination with HiveSessionProxy.invoke code, perhaps unintentionally, it used
- // to also catch all errors; and now it allows OOMs only to propagate.
- LOG.warn("Error executing statement: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetTypeInfoResp GetTypeInfo(TGetTypeInfoReq req) throws TException {
- TGetTypeInfoResp resp = new TGetTypeInfoResp();
- try {
- OperationHandle operationHandle = cliService.getTypeInfo(new SessionHandle(req.getSessionHandle()));
- resp.setOperationHandle(operationHandle.toTOperationHandle());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting type info: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetCatalogsResp GetCatalogs(TGetCatalogsReq req) throws TException {
- TGetCatalogsResp resp = new TGetCatalogsResp();
- try {
- OperationHandle opHandle = cliService.getCatalogs(new SessionHandle(req.getSessionHandle()));
- resp.setOperationHandle(opHandle.toTOperationHandle());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting catalogs: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetSchemasResp GetSchemas(TGetSchemasReq req) throws TException {
- TGetSchemasResp resp = new TGetSchemasResp();
- try {
- OperationHandle opHandle = cliService.getSchemas(
- new SessionHandle(req.getSessionHandle()), req.getCatalogName(), req.getSchemaName());
- resp.setOperationHandle(opHandle.toTOperationHandle());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting schemas: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetTablesResp GetTables(TGetTablesReq req) throws TException {
- TGetTablesResp resp = new TGetTablesResp();
- try {
- OperationHandle opHandle = cliService
- .getTables(new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
- req.getSchemaName(), req.getTableName(), req.getTableTypes());
- resp.setOperationHandle(opHandle.toTOperationHandle());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting tables: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetTableTypesResp GetTableTypes(TGetTableTypesReq req) throws TException {
- TGetTableTypesResp resp = new TGetTableTypesResp();
- try {
- OperationHandle opHandle = cliService.getTableTypes(new SessionHandle(req.getSessionHandle()));
- resp.setOperationHandle(opHandle.toTOperationHandle());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting table types: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetColumnsResp GetColumns(TGetColumnsReq req) throws TException {
- TGetColumnsResp resp = new TGetColumnsResp();
- try {
- OperationHandle opHandle = cliService.getColumns(
- new SessionHandle(req.getSessionHandle()),
- req.getCatalogName(),
- req.getSchemaName(),
- req.getTableName(),
- req.getColumnName());
- resp.setOperationHandle(opHandle.toTOperationHandle());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting columns: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetFunctionsResp GetFunctions(TGetFunctionsReq req) throws TException {
- TGetFunctionsResp resp = new TGetFunctionsResp();
- try {
- OperationHandle opHandle = cliService.getFunctions(
- new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
- req.getSchemaName(), req.getFunctionName());
- resp.setOperationHandle(opHandle.toTOperationHandle());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting functions: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetOperationStatusResp GetOperationStatus(TGetOperationStatusReq req) throws TException {
- TGetOperationStatusResp resp = new TGetOperationStatusResp();
- OperationHandle operationHandle = new OperationHandle(req.getOperationHandle());
- try {
- OperationStatus operationStatus =
- cliService.getOperationStatus(operationHandle, req.isGetProgressUpdate());
-
- resp.setOperationState(operationStatus.getState().toTOperationState());
- resp.setErrorMessage(operationStatus.getState().getErrorMessage());
- HiveSQLException opException = operationStatus.getOperationException();
- resp.setTaskStatus(operationStatus.getTaskStatus());
- resp.setOperationStarted(operationStatus.getOperationStarted());
- resp.setOperationCompleted(operationStatus.getOperationCompleted());
- resp.setHasResultSet(operationStatus.getHasResultSet());
- JobProgressUpdate progressUpdate = operationStatus.jobProgressUpdate();
- ProgressMonitorStatusMapper mapper = ProgressMonitorStatusMapper.DEFAULT;
-
- TJobExecutionStatus executionStatus =
- mapper.forStatus(progressUpdate.status);
- resp.setProgressUpdateResponse(new TProgressUpdateResp(
- progressUpdate.headers(),
- progressUpdate.rows(),
- progressUpdate.progressedPercentage,
- executionStatus,
- progressUpdate.footerSummary,
- progressUpdate.startTimeMillis
- ));
- if (opException != null) {
- resp.setSqlState(opException.getSQLState());
- resp.setErrorCode(opException.getErrorCode());
- if (opException.getErrorCode() == 29999)
- resp.setErrorMessage(org.apache.hadoop.util.StringUtils.stringifyException(opException));
- else
- resp.setErrorMessage(opException.getMessage());
- } else if (executionStatus == TJobExecutionStatus.NOT_AVAILABLE
- && OperationType.EXECUTE_STATEMENT.equals(operationHandle.getOperationType())) {
- resp.getProgressUpdateResponse().setProgressedPercentage(
- getProgressedPercentage(operationHandle));
- }
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting operation status: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TCancelOperationResp CancelOperation(TCancelOperationReq req) throws TException {
- TCancelOperationResp resp = new TCancelOperationResp();
- try {
- cliService.cancelOperation(new OperationHandle(req.getOperationHandle()));
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error cancelling operation: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TCloseOperationResp CloseOperation(TCloseOperationReq req) throws TException {
- TCloseOperationResp resp = new TCloseOperationResp();
- try {
- cliService.closeOperation(new OperationHandle(req.getOperationHandle()));
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error closing operation: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req)
- throws TException {
- TGetResultSetMetadataResp resp = new TGetResultSetMetadataResp();
- try {
- TableSchema schema = cliService.getResultSetMetadata(new OperationHandle(req.getOperationHandle()));
- resp.setSchema(schema.toTTableSchema());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting result set metadata: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TFetchResultsResp FetchResults(TFetchResultsReq req) throws TException {
- TFetchResultsResp resp = new TFetchResultsResp();
- try {
- // Set fetch size
- int maxFetchSize =
- hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_MAX_FETCH_SIZE);
- if (req.getMaxRows() > maxFetchSize) {
- req.setMaxRows(maxFetchSize);
- }
- RowSet rowSet = cliService.fetchResults(
- new OperationHandle(req.getOperationHandle()),
- FetchOrientation.getFetchOrientation(req.getOrientation()),
- req.getMaxRows(),
- FetchType.getFetchType(req.getFetchType()));
- resp.setResults(rowSet.toTRowSet());
- resp.setHasMoreRows(false);
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error fetching results: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetPrimaryKeysResp GetPrimaryKeys(TGetPrimaryKeysReq req)
- throws TException {
- TGetPrimaryKeysResp resp = new TGetPrimaryKeysResp();
- try {
- OperationHandle opHandle = cliService.getPrimaryKeys(
- new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
- req.getSchemaName(), req.getTableName());
- resp.setOperationHandle(opHandle.toTOperationHandle());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting functions: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req)
- throws TException {
- TGetCrossReferenceResp resp = new TGetCrossReferenceResp();
- try {
- OperationHandle opHandle = cliService.getCrossReference(
- new SessionHandle(req.getSessionHandle()), req.getParentCatalogName(),
- req.getParentSchemaName(), req.getParentTableName(),
- req.getForeignCatalogName(), req.getForeignSchemaName(), req.getForeignTableName());
- resp.setOperationHandle(opHandle.toTOperationHandle());
- resp.setStatus(OK_STATUS);
- } catch (Exception e) {
- LOG.warn("Error getting functions: ", e);
- resp.setStatus(HiveSQLException.toTStatus(e));
- }
- return resp;
- }
-
- @Override
- public TGetQueryIdResp GetQueryId(TGetQueryIdReq req) throws TException {
- try {
- return new TGetQueryIdResp(cliService.getQueryId(req.getOperationHandle()));
- } catch (HiveSQLException e) {
- throw new TException(e);
- }
- }
-
- @Override
- public abstract void run();
-
- /**
- * If the proxy user name is provided then check privileges to substitute the user.
- * @param realUser
- * @param sessionConf
- * @param ipAddress
- * @return
- * @throws HiveSQLException
- */
- private String getProxyUser(String realUser, Map<String, String> sessionConf,
- String ipAddress) throws HiveSQLException {
- String proxyUser = null;
- // Http transport mode.
- // We set the thread local proxy username, in ThriftHttpServlet.
- if (hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
- proxyUser = SessionInfo.getProxyUserName();
- LOG.debug("Proxy user from query string: " + proxyUser);
- }
-
- if (proxyUser == null && sessionConf != null && sessionConf.containsKey(HiveAuthConstants.HS2_PROXY_USER)) {
- String proxyUserFromThriftBody = sessionConf.get(HiveAuthConstants.HS2_PROXY_USER);
- LOG.debug("Proxy user from thrift body: " + proxyUserFromThriftBody);
- proxyUser = proxyUserFromThriftBody;
- }
-
- if (proxyUser == null) {
- return realUser;
- }
-
- // check whether substitution is allowed
- if (!hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ALLOW_USER_SUBSTITUTION)) {
- throw new HiveSQLException("Proxy user substitution is not allowed");
- }
-
- // If there's no authentication, then directly substitute the user
- if (HiveAuthConstants.AuthTypes.NONE.toString().
- equalsIgnoreCase(hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION))) {
- return proxyUser;
- }
-
- // Verify proxy user privilege of the realUser for the proxyUser
- HiveAuthFactory.verifyProxyAccess(realUser, proxyUser, ipAddress, hiveConf);
- LOG.debug("Verified proxy user: " + proxyUser);
- return proxyUser;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
deleted file mode 100644
index 4e48cc5..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import java.util.Arrays;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.TimeUnit;
-
-import javax.ws.rs.HttpMethod;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.ICLIService;
-import org.apache.hive.service.rpc.thrift.TCLIService;
-import org.apache.hive.service.rpc.thrift.TCLIService.Iface;
-import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
-import org.apache.livy.thriftserver.LivyCLIService;
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TProtocolFactory;
-import org.apache.thrift.server.TServlet;
-import org.eclipse.jetty.server.HttpConfiguration;
-import org.eclipse.jetty.server.HttpConnectionFactory;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.server.handler.gzip.GzipHandler;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.eclipse.jetty.util.thread.ExecutorThreadPool;
-
-
-public class ThriftHttpCLIService extends ThriftCLIService {
- private static final String APPLICATION_THRIFT = "application/x-thrift";
- protected org.eclipse.jetty.server.Server server;
-
- private final Runnable oomHook;
- public ThriftHttpCLIService(ICLIService cliService, Runnable oomHook) {
- super(cliService, ThriftHttpCLIService.class.getSimpleName());
- this.oomHook = oomHook;
- }
-
- /**
- * Configure Jetty to serve http requests. Example of a client connection URL:
- * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target
- * URL to differ, e.g. http://gateway:port/hive2/servlets/thrifths2/
- */
- @Override
- protected void initServer() {
- try {
- // Server thread pool
- // Start with minWorkerThreads, expand till maxWorkerThreads and reject
- // subsequent requests
- String threadPoolName = "HiveServer2-HttpHandler-Pool";
- ExecutorService executorService = new ThreadPoolExecutorWithOomHook(minWorkerThreads,
- maxWorkerThreads,workerKeepAliveTime, TimeUnit.SECONDS,
- new SynchronousQueue<Runnable>(), new ThreadFactoryWithGarbageCleanup(threadPoolName), oomHook);
- ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService);
-
- // HTTP Server
- server = new Server(threadPool);
-
- ServerConnector connector;
-
- final HttpConfiguration conf = new HttpConfiguration();
- // Configure header size
- int requestHeaderSize =
- hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_REQUEST_HEADER_SIZE);
- int responseHeaderSize =
- hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_RESPONSE_HEADER_SIZE);
- conf.setRequestHeaderSize(requestHeaderSize);
- conf.setResponseHeaderSize(responseHeaderSize);
- final HttpConnectionFactory http = new HttpConnectionFactory(conf);
-
- boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL);
- String schemeName = useSsl ? "https" : "http";
-
- // Change connector if SSL is used
- if (useSsl) {
- String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim();
- String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf,
- HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname);
- if (keyStorePath.isEmpty()) {
- throw new IllegalArgumentException(
- ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname
- + " Not configured for SSL connection");
- }
- SslContextFactory sslContextFactory = new SslContextFactory();
- String[] excludedProtocols = hiveConf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST).split(",");
- LOG.info("HTTP Server SSL: adding excluded protocols: " + Arrays.toString(excludedProtocols));
- sslContextFactory.addExcludeProtocols(excludedProtocols);
- LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = "
- + Arrays.toString(sslContextFactory.getExcludeProtocols()));
- sslContextFactory.setKeyStorePath(keyStorePath);
- sslContextFactory.setKeyStorePassword(keyStorePassword);
- connector = new ServerConnector(server, sslContextFactory, http);
- } else {
- connector = new ServerConnector(server, http);
- }
-
- connector.setPort(portNum);
- // Linux:yes, Windows:no
- connector.setReuseAddress(true);
- int maxIdleTime = (int) hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME,
- TimeUnit.MILLISECONDS);
- connector.setIdleTimeout(maxIdleTime);
-
- server.addConnector(connector);
-
- // Thrift configs
- hiveAuthFactory = new HiveAuthFactory(hiveConf);
- TProcessor processor = new TCLIService.Processor<Iface>(this);
- TProtocolFactory protocolFactory = new TBinaryProtocol.Factory();
- // Set during the init phase of HiveServer2 if auth mode is kerberos
- // UGI for the hive/_HOST (kerberos) principal
- UserGroupInformation serviceUGI = ((LivyCLIService) cliService).getServiceUGI();
- // UGI for the http/_HOST (SPNego) principal
- UserGroupInformation httpUGI = ((LivyCLIService) cliService).getHttpUGI();
- String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION);
- TServlet thriftHttpServlet = new ThriftHttpServlet(processor, protocolFactory, authType, serviceUGI, httpUGI,
- hiveAuthFactory);
-
- // Context handler
- final ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
- context.setContextPath("/");
- if (hiveConf.getBoolean(ConfVars.HIVE_SERVER2_XSRF_FILTER_ENABLED.varname, false)) {
- // context.addFilter(Utils.getXSRFFilterHolder(null, null), "/" ,
- // FilterMapping.REQUEST);
- // Filtering does not work here currently, doing filter in ThriftHttpServlet
- LOG.debug("XSRF filter enabled");
- } else {
- LOG.warn("XSRF filter disabled");
- }
-
- final String httpPath = getHttpPath(hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH));
-
- if (HiveConf.getBoolVar(hiveConf, ConfVars.HIVE_SERVER2_THRIFT_HTTP_COMPRESSION_ENABLED)) {
- final GzipHandler gzipHandler = new GzipHandler();
- gzipHandler.setHandler(context);
- gzipHandler.addIncludedMethods(HttpMethod.POST);
- gzipHandler.addIncludedMimeTypes(APPLICATION_THRIFT);
- server.setHandler(gzipHandler);
- } else {
- server.setHandler(context);
- }
- context.addServlet(new ServletHolder(thriftHttpServlet), httpPath);
-
- // TODO: check defaults: maxTimeout, keepalive, maxBodySize,
- // bodyRecieveDuration, etc.
- // Finally, start the server
- server.start();
- String msg = "Started " + ThriftHttpCLIService.class.getSimpleName() + " in " + schemeName
- + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..."
- + maxWorkerThreads + " worker threads";
- LOG.info(msg);
- } catch (Exception e) {
- throw new RuntimeException("Failed to init HttpServer", e);
- }
- }
-
- @Override
- public void run() {
- try {
- server.join();
- } catch (Throwable t) {
- if (t instanceof InterruptedException) {
- // This is likely a shutdown
- LOG.info("Caught " + t.getClass().getSimpleName() + ". Shutting down thrift server.");
- } else {
- LOG.error("Exception caught by " + ThriftHttpCLIService.class.getSimpleName() +
- ". Exiting.", t);
- System.exit(-1);
- }
- }
- }
-
- /**
- * The config parameter can be like "path", "/path", "/path/", "path/*", "/path1/path2/*" and so on.
- * httpPath should end up as "/*", "/path/*" or "/path1/../pathN/*"
- * @param httpPath
- * @return
- */
- private String getHttpPath(String httpPath) {
- if(httpPath == null || httpPath.equals("")) {
- httpPath = "/*";
- }
- else {
- if(!httpPath.startsWith("/")) {
- httpPath = "/" + httpPath;
- }
- if(httpPath.endsWith("/")) {
- httpPath = httpPath + "*";
- }
- if(!httpPath.endsWith("/*")) {
- httpPath = httpPath + "/*";
- }
- }
- return httpPath;
- }
-
- @Override
- protected void stopServer() {
- if((server != null) && server.isStarted()) {
- try {
- server.stop();
- server = null;
- LOG.info("Thrift HTTP server has been stopped");
- } catch (Exception e) {
- LOG.error("Error stopping HTTP server: ", e);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
deleted file mode 100644
index 33c942f..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
+++ /dev/null
@@ -1,592 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.security.PrivilegedExceptionAction;
-import java.security.SecureRandom;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.Cookie;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.core.NewCookie;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.StringUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.service.CookieSigner;
-import org.apache.hive.service.auth.AuthenticationProviderFactory;
-import org.apache.hive.service.auth.AuthenticationProviderFactory.AuthMethods;
-import org.apache.hive.service.auth.HiveAuthConstants;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.auth.HttpAuthUtils;
-import org.apache.hive.service.auth.HttpAuthenticationException;
-import org.apache.hive.service.auth.PasswdAuthenticationProvider;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TProtocolFactory;
-import org.apache.thrift.server.TServlet;
-import org.ietf.jgss.GSSContext;
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
-import org.ietf.jgss.GSSManager;
-import org.ietf.jgss.GSSName;
-import org.ietf.jgss.Oid;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.livy.thriftserver.SessionInfo;
-
-/**
- *
- * ThriftHttpServlet
- *
- */
-public class ThriftHttpServlet extends TServlet {
-
- private static final long serialVersionUID = 1L;
- public static final Logger LOG = LoggerFactory.getLogger(ThriftHttpServlet.class.getName());
- private final String authType;
- private final UserGroupInformation serviceUGI;
- private final UserGroupInformation httpUGI;
- private HiveConf hiveConf = new HiveConf();
-
- // Class members for cookie based authentication.
- private CookieSigner signer;
- public static final String AUTH_COOKIE = "hive.server2.auth";
- private static final SecureRandom RAN = new SecureRandom();
- private boolean isCookieAuthEnabled;
- private String cookieDomain;
- private String cookiePath;
- private int cookieMaxAge;
- private boolean isCookieSecure;
- private boolean isHttpOnlyCookie;
- private final HiveAuthFactory hiveAuthFactory;
- private static final String HIVE_DELEGATION_TOKEN_HEADER = "X-Hive-Delegation-Token";
- private static final String X_FORWARDED_FOR = "X-Forwarded-For";
-
- public ThriftHttpServlet(TProcessor processor, TProtocolFactory protocolFactory,
- String authType, UserGroupInformation serviceUGI, UserGroupInformation httpUGI,
- HiveAuthFactory hiveAuthFactory) {
- super(processor, protocolFactory);
- this.authType = authType;
- this.serviceUGI = serviceUGI;
- this.httpUGI = httpUGI;
- this.hiveAuthFactory = hiveAuthFactory;
- this.isCookieAuthEnabled = hiveConf.getBoolVar(
- ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED);
- // Initialize the cookie based authentication related variables.
- if (isCookieAuthEnabled) {
- // Generate the signer with secret.
- String secret = Long.toString(RAN.nextLong());
- LOG.debug("Using the random number as the secret for cookie generation " + secret);
- this.signer = new CookieSigner(secret.getBytes());
- this.cookieMaxAge = (int) hiveConf.getTimeVar(
- ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE, TimeUnit.SECONDS);
- this.cookieDomain = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_DOMAIN);
- this.cookiePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_PATH);
- // always send secure cookies for SSL mode
- this.isCookieSecure = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL);
- this.isHttpOnlyCookie = hiveConf.getBoolVar(
- ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_HTTPONLY);
- }
- }
-
- @Override
- protected void doPost(HttpServletRequest request, HttpServletResponse response)
- throws ServletException, IOException {
- String clientUserName = null;
- String clientIpAddress;
- boolean requireNewCookie = false;
-
- try {
- if (hiveConf.getBoolean(ConfVars.HIVE_SERVER2_XSRF_FILTER_ENABLED.varname,false)){
- boolean continueProcessing = Utils.doXsrfFilter(request,response,null,null);
- if (!continueProcessing){
- LOG.warn("Request did not have valid XSRF header, rejecting.");
- return;
- }
- }
- // If the cookie based authentication is already enabled, parse the
- // request and validate the request cookies.
- if (isCookieAuthEnabled) {
- clientUserName = validateCookie(request);
- requireNewCookie = (clientUserName == null);
- if (requireNewCookie) {
- LOG.info("Could not validate cookie sent, will try to generate a new cookie");
- }
- }
- // If the cookie based authentication is not enabled or the request does
- // not have a valid cookie, use the kerberos or password based authentication
- // depending on the server setup.
- if (clientUserName == null) {
- // For a kerberos setup
- if (isKerberosAuthMode(authType)) {
- String delegationToken = request.getHeader(HIVE_DELEGATION_TOKEN_HEADER);
- // Each http request must have an Authorization header
- if ((delegationToken != null) && (!delegationToken.isEmpty())) {
- clientUserName = doTokenAuth(request, response);
- } else {
- clientUserName = doKerberosAuth(request);
- }
- }
- // For password based authentication
- else {
- clientUserName = doPasswdAuth(request, authType);
- }
- }
- LOG.debug("Client username: " + clientUserName);
-
- // Set the thread local username to be used for doAs if true
- SessionInfo.setUserName(clientUserName);
-
- // find proxy user if any from query param
- String doAsQueryParam = getDoAsQueryParam(request.getQueryString());
- if (doAsQueryParam != null) {
- SessionInfo.setProxyUserName(doAsQueryParam);
- }
-
- clientIpAddress = request.getRemoteAddr();
- LOG.debug("Client IP Address: " + clientIpAddress);
- // Set the thread local ip address
- SessionInfo.setIpAddress(clientIpAddress);
-
- // get forwarded hosts address
- String forwarded_for = request.getHeader(X_FORWARDED_FOR);
- if (forwarded_for != null) {
- LOG.debug("{}:{}", X_FORWARDED_FOR, forwarded_for);
- List<String> forwardedAddresses = Arrays.asList(forwarded_for.split(","));
- SessionInfo.setForwardedAddresses(forwardedAddresses);
- } else {
- SessionInfo.setForwardedAddresses(Collections.<String>emptyList());
- }
-
- // Generate new cookie and add it to the response
- if (requireNewCookie &&
- !authType.equalsIgnoreCase(HiveAuthConstants.AuthTypes.NOSASL.toString())) {
- String cookieToken = HttpAuthUtils.createCookieToken(clientUserName);
- Cookie hs2Cookie = createCookie(signer.signCookie(cookieToken));
-
- if (isHttpOnlyCookie) {
- response.setHeader("SET-COOKIE", getHttpOnlyCookieHeader(hs2Cookie));
- } else {
- response.addCookie(hs2Cookie);
- }
- LOG.info("Cookie added for clientUserName " + clientUserName);
- }
- super.doPost(request, response);
- }
- catch (HttpAuthenticationException e) {
- LOG.error("Error: ", e);
- // Send a 401 to the client
- response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
- if(isKerberosAuthMode(authType)) {
- response.addHeader(HttpAuthUtils.WWW_AUTHENTICATE, HttpAuthUtils.NEGOTIATE);
- }
- response.getWriter().println("Authentication Error: " + e.getMessage());
- }
- finally {
- // Clear the thread locals
- SessionInfo.clearUserName();
- SessionInfo.clearIpAddress();
- SessionInfo.clearProxyUserName();
- SessionInfo.clearForwardedAddresses();
- }
- }
-
- /**
- * Retrieves the client name from cookieString. If the cookie does not
- * correspond to a valid client, the function returns null.
- * @param cookies HTTP Request cookies.
- * @return Client Username if cookieString has a HS2 Generated cookie that is currently valid.
- * Else, returns null.
- */
- private String getClientNameFromCookie(Cookie[] cookies) {
- // Current Cookie Name, Current Cookie Value
- String currName, currValue;
-
- // Following is the main loop which iterates through all the cookies send by the client.
- // The HS2 generated cookies are of the format hive.server2.auth=<value>
- // A cookie which is identified as a hiveserver2 generated cookie is validated
- // by calling signer.verifyAndExtract(). If the validation passes, send the
- // username for which the cookie is validated to the caller. If no client side
- // cookie passes the validation, return null to the caller.
- for (Cookie currCookie : cookies) {
- // Get the cookie name
- currName = currCookie.getName();
- if (!currName.equals(AUTH_COOKIE)) {
- // Not a HS2 generated cookie, continue.
- continue;
- }
- // If we reached here, we have match for HS2 generated cookie
- currValue = currCookie.getValue();
- // Validate the value.
- currValue = signer.verifyAndExtract(currValue);
- // Retrieve the user name, do the final validation step.
- if (currValue != null) {
- String userName = HttpAuthUtils.getUserNameFromCookieToken(currValue);
-
- if (userName == null) {
- LOG.warn("Invalid cookie token " + currValue);
- continue;
- }
- //We have found a valid cookie in the client request.
- if (LOG.isDebugEnabled()) {
- LOG.debug("Validated the cookie for user " + userName);
- }
- return userName;
- }
- }
- // No valid HS2 generated cookies found, return null
- return null;
- }
-
- /**
- * Convert cookie array to human readable cookie string
- * @param cookies Cookie Array
- * @return String containing all the cookies separated by a newline character.
- * Each cookie is of the format [key]=[value]
- */
- private String toCookieStr(Cookie[] cookies) {
- String cookieStr = "";
-
- for (Cookie c : cookies) {
- cookieStr += c.getName() + "=" + c.getValue() + " ;\n";
- }
- return cookieStr;
- }
-
- /**
- * Validate the request cookie. This function iterates over the request cookie headers
- * and finds a cookie that represents a valid client/server session. If it finds one, it
- * returns the client name associated with the session. Else, it returns null.
- * @param request The HTTP Servlet Request send by the client
- * @return Client Username if the request has valid HS2 cookie, else returns null
- * @throws UnsupportedEncodingException
- */
- private String validateCookie(HttpServletRequest request) throws UnsupportedEncodingException {
- // Find all the valid cookies associated with the request.
- Cookie[] cookies = request.getCookies();
-
- if (cookies == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No valid cookies associated with the request " + request);
- }
- return null;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Received cookies: " + toCookieStr(cookies));
- }
- return getClientNameFromCookie(cookies);
- }
-
- /**
- * Generate a server side cookie given the cookie value as the input.
- * @param str Input string token.
- * @return The generated cookie.
- * @throws UnsupportedEncodingException
- */
- private Cookie createCookie(String str) throws UnsupportedEncodingException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Cookie name = " + AUTH_COOKIE + " value = " + str);
- }
- Cookie cookie = new Cookie(AUTH_COOKIE, str);
-
- cookie.setMaxAge(cookieMaxAge);
- if (cookieDomain != null) {
- cookie.setDomain(cookieDomain);
- }
- if (cookiePath != null) {
- cookie.setPath(cookiePath);
- }
- cookie.setSecure(isCookieSecure);
- return cookie;
- }
-
- /**
- * Generate httponly cookie from HS2 cookie
- * @param cookie HS2 generated cookie
- * @return The httponly cookie
- */
- private static String getHttpOnlyCookieHeader(Cookie cookie) {
- NewCookie newCookie = new NewCookie(cookie.getName(), cookie.getValue(),
- cookie.getPath(), cookie.getDomain(), cookie.getVersion(),
- cookie.getComment(), cookie.getMaxAge(), cookie.getSecure());
- return newCookie + "; HttpOnly";
- }
-
- /**
- * Do the LDAP/PAM authentication
- * @param request
- * @param authType
- * @throws HttpAuthenticationException
- */
- private String doPasswdAuth(HttpServletRequest request, String authType)
- throws HttpAuthenticationException {
- String userName = getUsername(request, authType);
- // No-op when authType is NOSASL
- if (!authType.equalsIgnoreCase(HiveAuthConstants.AuthTypes.NOSASL.toString())) {
- try {
- AuthMethods authMethod = AuthMethods.getValidAuthMethod(authType);
- PasswdAuthenticationProvider provider =
- AuthenticationProviderFactory.getAuthenticationProvider(authMethod, hiveConf);
- provider.Authenticate(userName, getPassword(request, authType));
-
- } catch (Exception e) {
- throw new HttpAuthenticationException(e);
- }
- }
- return userName;
- }
-
- private String doTokenAuth(HttpServletRequest request, HttpServletResponse response)
- throws HttpAuthenticationException {
- String tokenStr = request.getHeader(HIVE_DELEGATION_TOKEN_HEADER);
- try {
- return hiveAuthFactory.verifyDelegationToken(tokenStr);
- } catch (HiveSQLException e) {
- throw new HttpAuthenticationException(e);
- }
- }
-
- /**
- * Do the GSS-API kerberos authentication.
- * We already have a logged in subject in the form of serviceUGI,
- * which GSS-API will extract information from.
- * In case of a SPNego request we use the httpUGI,
- * for the authenticating service tickets.
- * @param request
- * @return
- * @throws HttpAuthenticationException
- */
- private String doKerberosAuth(HttpServletRequest request)
- throws HttpAuthenticationException {
- // Try authenticating with the http/_HOST principal
- if (httpUGI != null) {
- try {
- return httpUGI.doAs(new HttpKerberosServerAction(request, httpUGI));
- } catch (Exception e) {
- LOG.info("Failed to authenticate with http/_HOST kerberos principal, " +
- "trying with hive/_HOST kerberos principal");
- }
- }
- // Now try with hive/_HOST principal
- try {
- return serviceUGI.doAs(new HttpKerberosServerAction(request, serviceUGI));
- } catch (Exception e) {
- LOG.error("Failed to authenticate with hive/_HOST kerberos principal");
- throw new HttpAuthenticationException(e);
- }
-
- }
-
- class HttpKerberosServerAction implements PrivilegedExceptionAction<String> {
- HttpServletRequest request;
- UserGroupInformation serviceUGI;
-
- HttpKerberosServerAction(HttpServletRequest request,
- UserGroupInformation serviceUGI) {
- this.request = request;
- this.serviceUGI = serviceUGI;
- }
-
- @Override
- public String run() throws HttpAuthenticationException {
- // Get own Kerberos credentials for accepting connection
- GSSManager manager = GSSManager.getInstance();
- GSSContext gssContext = null;
- String serverPrincipal = getPrincipalWithoutRealm(
- serviceUGI.getUserName());
- try {
- // This Oid for Kerberos GSS-API mechanism.
- Oid kerberosMechOid = new Oid("1.2.840.113554.1.2.2");
- // Oid for SPNego GSS-API mechanism.
- Oid spnegoMechOid = new Oid("1.3.6.1.5.5.2");
- // Oid for kerberos principal name
- Oid krb5PrincipalOid = new Oid("1.2.840.113554.1.2.2.1");
-
- // GSS name for server
- GSSName serverName = manager.createName(serverPrincipal, krb5PrincipalOid);
-
- // GSS credentials for server
- GSSCredential serverCreds = manager.createCredential(serverName,
- GSSCredential.DEFAULT_LIFETIME,
- new Oid[]{kerberosMechOid, spnegoMechOid},
- GSSCredential.ACCEPT_ONLY);
-
- // Create a GSS context
- gssContext = manager.createContext(serverCreds);
- // Get service ticket from the authorization header
- String serviceTicketBase64 = getAuthHeader(request, authType);
- byte[] inToken = Base64.decodeBase64(serviceTicketBase64.getBytes());
- gssContext.acceptSecContext(inToken, 0, inToken.length);
- // Authenticate or deny based on its context completion
- if (!gssContext.isEstablished()) {
- throw new HttpAuthenticationException("Kerberos authentication failed: " +
- "unable to establish context with the service ticket " +
- "provided by the client.");
- }
- else {
- return getPrincipalWithoutRealmAndHost(gssContext.getSrcName().toString());
- }
- }
- catch (GSSException e) {
- throw new HttpAuthenticationException("Kerberos authentication failed: ", e);
- }
- finally {
- if (gssContext != null) {
- try {
- gssContext.dispose();
- } catch (GSSException e) {
- // No-op
- }
- }
- }
- }
-
- private String getPrincipalWithoutRealm(String fullPrincipal)
- throws HttpAuthenticationException {
- KerberosNameShim fullKerberosName;
- try {
- fullKerberosName = ShimLoader.getHadoopShims().getKerberosNameShim(fullPrincipal);
- } catch (IOException e) {
- throw new HttpAuthenticationException(e);
- }
- String serviceName = fullKerberosName.getServiceName();
- String hostName = fullKerberosName.getHostName();
- String principalWithoutRealm = serviceName;
- if (hostName != null) {
- principalWithoutRealm = serviceName + "/" + hostName;
- }
- return principalWithoutRealm;
- }
-
- private String getPrincipalWithoutRealmAndHost(String fullPrincipal)
- throws HttpAuthenticationException {
- KerberosNameShim fullKerberosName;
- try {
- fullKerberosName = ShimLoader.getHadoopShims().getKerberosNameShim(fullPrincipal);
- return fullKerberosName.getShortName();
- } catch (IOException e) {
- throw new HttpAuthenticationException(e);
- }
- }
- }
-
- private String getUsername(HttpServletRequest request, String authType)
- throws HttpAuthenticationException {
- String creds[] = getAuthHeaderTokens(request, authType);
- // Username must be present
- if (creds[0] == null || creds[0].isEmpty()) {
- throw new HttpAuthenticationException("Authorization header received " +
- "from the client does not contain username.");
- }
- return creds[0];
- }
-
- private String getPassword(HttpServletRequest request, String authType)
- throws HttpAuthenticationException {
- String creds[] = getAuthHeaderTokens(request, authType);
- // Password must be present
- if (creds[1] == null || creds[1].isEmpty()) {
- throw new HttpAuthenticationException("Authorization header received " +
- "from the client does not contain username.");
- }
- return creds[1];
- }
-
- private String[] getAuthHeaderTokens(HttpServletRequest request,
- String authType) throws HttpAuthenticationException {
- String authHeaderBase64 = getAuthHeader(request, authType);
- String authHeaderString = StringUtils.newStringUtf8(
- Base64.decodeBase64(authHeaderBase64.getBytes()));
- String[] creds = authHeaderString.split(":");
- return creds;
- }
-
- /**
- * Returns the base64 encoded auth header payload
- * @param request
- * @param authType
- * @return
- * @throws HttpAuthenticationException
- */
- private String getAuthHeader(HttpServletRequest request, String authType)
- throws HttpAuthenticationException {
- String authHeader = request.getHeader(HttpAuthUtils.AUTHORIZATION);
- // Each http request must have an Authorization header
- if (authHeader == null || authHeader.isEmpty()) {
- throw new HttpAuthenticationException("Authorization header received " +
- "from the client is empty.");
- }
-
- String authHeaderBase64String;
- int beginIndex;
- if (isKerberosAuthMode(authType)) {
- beginIndex = (HttpAuthUtils.NEGOTIATE + " ").length();
- }
- else {
- beginIndex = (HttpAuthUtils.BASIC + " ").length();
- }
- authHeaderBase64String = authHeader.substring(beginIndex);
- // Authorization header must have a payload
- if (authHeaderBase64String == null || authHeaderBase64String.isEmpty()) {
- throw new HttpAuthenticationException("Authorization header received " +
- "from the client does not contain any data.");
- }
- return authHeaderBase64String;
- }
-
- private boolean isKerberosAuthMode(String authType) {
- return authType.equalsIgnoreCase(HiveAuthConstants.AuthTypes.KERBEROS.toString());
- }
-
- private static String getDoAsQueryParam(String queryString) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("URL query string:" + queryString);
- }
- if (queryString == null) {
- return null;
- }
- Map<String, String[]> params = javax.servlet.http.HttpUtils.parseQueryString( queryString );
- Set<String> keySet = params.keySet();
- for (String key: keySet) {
- if (key.equalsIgnoreCase("doAs")) {
- return params.get(key)[0];
- }
- }
- return null;
- }
-
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/server/HiveServer2.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/server/HiveServer2.java b/thriftserver/server/src/main/java/org/apache/hive/service/server/HiveServer2.java
deleted file mode 100644
index 3db0590..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/server/HiveServer2.java
+++ /dev/null
@@ -1,316 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.server;
-
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.JvmPauseMonitor;
-import org.apache.hadoop.hive.common.LogUtils;
-import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
-import org.apache.hadoop.hive.common.cli.CommonCliOptions;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hive.common.util.HiveStringUtils;
-import org.apache.hive.common.util.ShutdownHookManager;
-import org.apache.hive.service.CompositeService;
-import org.apache.hive.service.cli.ICLIService;
-import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService;
-import org.apache.hive.service.cli.thrift.ThriftCLIService;
-import org.apache.hive.service.cli.thrift.ThriftHttpCLIService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * HiveServer2.
- *
- */
-public class HiveServer2 extends CompositeService {
- private static final Logger LOG = LoggerFactory.getLogger(HiveServer2.class);
- public static final String INSTANCE_URI_CONFIG = "hive.server2.instance.uri";
- protected ICLIService cliService;
- protected ThriftCLIService thriftCLIService;
-
- public HiveServer2() {
- super(HiveServer2.class.getSimpleName());
- HiveConf.setLoadHiveServer2Config(true);
- }
-
- @Override
- public synchronized void init(HiveConf hiveConf) {
- assert cliService != null;
- assert cliService instanceof CompositeService;
- addService((CompositeService) cliService);
- final HiveServer2 hiveServer2 = this;
- Runnable oomHook = new Runnable() {
- @Override
- public void run() {
- hiveServer2.stop();
- }
- };
- if (isHTTPTransportMode(hiveConf)) {
- thriftCLIService = new ThriftHttpCLIService(cliService, oomHook);
- } else {
- thriftCLIService = new ThriftBinaryCLIService(cliService, oomHook);
- }
- addService(thriftCLIService);
- super.init(hiveConf);
- // Add a shutdown hook for catching SIGTERM & SIGINT
- ShutdownHookManager.addShutdownHook(new Thread() {
- @Override
- public void run() {
- hiveServer2.stop();
- }
- });
- }
-
- public static boolean isHTTPTransportMode(Configuration hiveConf) {
- String transportMode = System.getenv("HIVE_SERVER2_TRANSPORT_MODE");
- if (transportMode == null) {
- transportMode = hiveConf.get(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname);
- }
- if (transportMode != null && (transportMode.equalsIgnoreCase("http"))) {
- return true;
- }
- return false;
- }
-
- public static boolean isKerberosAuthMode(Configuration hiveConf) {
- String authMode = hiveConf.get(ConfVars.HIVE_SERVER2_AUTHENTICATION.varname);
- if (authMode != null && (authMode.equalsIgnoreCase("KERBEROS"))) {
- return true;
- }
- return false;
- }
-
- public String getServerHost() throws Exception {
- if ((thriftCLIService == null) || (thriftCLIService.getServerIPAddress() == null)) {
- throw new Exception("Unable to get the server address; it hasn't been initialized yet.");
- }
- return thriftCLIService.getServerIPAddress().getHostName();
- }
-
- @Override
- public synchronized void start() {
- super.start();
- }
-
- @Override
- public synchronized void stop() {
- LOG.info("Shutting down HiveServer2");
- super.stop();
- }
-
- private static void startHiveServer2() throws Throwable {
- long attempts = 0, maxAttempts = 1;
- while (true) {
- LOG.info("Starting HiveServer2");
- HiveConf hiveConf = new HiveConf();
- maxAttempts = hiveConf.getLongVar(HiveConf.ConfVars.HIVE_SERVER2_MAX_START_ATTEMPTS);
- long retrySleepIntervalMs = hiveConf
- .getTimeVar(ConfVars.HIVE_SERVER2_SLEEP_INTERVAL_BETWEEN_START_ATTEMPTS,
- TimeUnit.MILLISECONDS);
- HiveServer2 server = null;
- try {
- server = new HiveServer2();
- server.init(hiveConf);
- server.start();
-
- try {
- JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(hiveConf);
- pauseMonitor.start();
- } catch (Throwable t) {
- LOG.warn("Could not initiate the JvmPauseMonitor thread." + " GCs and Pauses may not be " +
- "warned upon.", t);
- }
- break;
- } catch (Throwable throwable) {
- if (server != null) {
- try {
- server.stop();
- } catch (Throwable t) {
- LOG.info("Exception caught when calling stop of HiveServer2 before retrying start", t);
- } finally {
- server = null;
- }
- }
- if (++attempts >= maxAttempts) {
- throw new Error("Max start attempts " + maxAttempts + " exhausted", throwable);
- } else {
- LOG.warn("Error starting HiveServer2 on attempt " + attempts
- + ", will retry in " + retrySleepIntervalMs + "ms", throwable);
- try {
- Thread.sleep(retrySleepIntervalMs);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
- }
- }
- }
-
- public static void main(String[] args) {
- HiveConf.setLoadHiveServer2Config(true);
- try {
- ServerOptionsProcessor oproc = new ServerOptionsProcessor("hiveserver2");
- ServerOptionsProcessorResponse oprocResponse = oproc.parse(args);
-
- // NOTE: It is critical to do this here so that log4j is reinitialized
- // before any of the other core hive classes are loaded
- String initLog4jMessage = LogUtils.initHiveLog4j();
- LOG.debug(initLog4jMessage);
- HiveStringUtils.startupShutdownMessage(HiveServer2.class, args, LOG);
-
- // Logger debug message from "oproc" after log4j initialize properly
- LOG.debug(oproc.getDebugMessage().toString());
-
- // Call the executor which will execute the appropriate command based on the parsed options
- oprocResponse.getServerOptionsExecutor().execute();
- } catch (LogInitializationException e) {
- LOG.error("Error initializing log: " + e.getMessage(), e);
- System.exit(-1);
- }
- }
-
- /**
- * ServerOptionsProcessor.
- * Process arguments given to HiveServer2 (-hiveconf property=value)
- * Set properties in System properties
- * Create an appropriate response object,
- * which has executor to execute the appropriate command based on the parsed options.
- */
- public static class ServerOptionsProcessor {
- private final Options options = new Options();
- private org.apache.commons.cli.CommandLine commandLine;
- private final String serverName;
- private final StringBuilder debugMessage = new StringBuilder();
-
- @SuppressWarnings("static-access")
- public ServerOptionsProcessor(String serverName) {
- this.serverName = serverName;
- // -hiveconf x=y
- options.addOption(OptionBuilder
- .withValueSeparator()
- .hasArgs(2)
- .withArgName("property=value")
- .withLongOpt("hiveconf")
- .withDescription("Use value for given property")
- .create());
- options.addOption(new Option("H", "help", false, "Print help information"));
- }
-
- public ServerOptionsProcessorResponse parse(String[] argv) {
- try {
- commandLine = new GnuParser().parse(options, argv);
- // Process --hiveconf
- // Get hiveconf param values and set the System property values
- Properties confProps = commandLine.getOptionProperties("hiveconf");
- for (String propKey : confProps.stringPropertyNames()) {
- // save logging message for log4j output latter after log4j initialize properly
- debugMessage.append("Setting " + propKey + "=" + confProps.getProperty(propKey) + ";\n");
- if (propKey.equalsIgnoreCase("hive.root.logger")) {
- CommonCliOptions.splitAndSetLogger(propKey, confProps);
- } else {
- System.setProperty(propKey, confProps.getProperty(propKey));
- }
- }
-
- // Process --help
- if (commandLine.hasOption('H')) {
- return new ServerOptionsProcessorResponse(new HelpOptionExecutor(serverName, options));
- }
- } catch (ParseException e) {
- // Error out & exit - we were not able to parse the args successfully
- System.err.println("Error starting HiveServer2 with given arguments: ");
- System.err.println(e.getMessage());
- System.exit(-1);
- }
- // Default executor, when no option is specified
- return new ServerOptionsProcessorResponse(new StartOptionExecutor());
- }
-
- StringBuilder getDebugMessage() {
- return debugMessage;
- }
- }
-
- /**
- * The response sent back from {@link ServerOptionsProcessor#parse(String[])}
- */
- static class ServerOptionsProcessorResponse {
- private final ServerOptionsExecutor serverOptionsExecutor;
-
- ServerOptionsProcessorResponse(ServerOptionsExecutor serverOptionsExecutor) {
- this.serverOptionsExecutor = serverOptionsExecutor;
- }
-
- ServerOptionsExecutor getServerOptionsExecutor() {
- return serverOptionsExecutor;
- }
- }
-
- /**
- * The executor interface for running the appropriate HiveServer2 command based on parsed options
- */
- static interface ServerOptionsExecutor {
- public void execute();
- }
-
- /**
- * HelpOptionExecutor: executes the --help option by printing out the usage
- */
- static class HelpOptionExecutor implements ServerOptionsExecutor {
- private final Options options;
- private final String serverName;
-
- HelpOptionExecutor(String serverName, Options options) {
- this.options = options;
- this.serverName = serverName;
- }
-
- @Override
- public void execute() {
- new HelpFormatter().printHelp(serverName, options);
- System.exit(0);
- }
- }
-
- /**
- * StartOptionExecutor: starts HiveServer2.
- * This is the default executor, when no option is specified.
- */
- static class StartOptionExecutor implements ServerOptionsExecutor {
- @Override
- public void execute() {
- try {
- startHiveServer2();
- } catch (Throwable t) {
- LOG.error("Error starting HiveServer2", t);
- System.exit(-1);
- }
- }
- }
-}
[2/5] incubator-livy git commit: [LIVY-502] Remove dependency on
hive-exec
Posted by va...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftBinaryCLIService.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftBinaryCLIService.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftBinaryCLIService.scala
new file mode 100644
index 0000000..e16313d
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftBinaryCLIService.scala
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.cli
+
+import java.net.InetSocketAddress
+import java.util
+import java.util.concurrent._
+import javax.net.ssl.SSLServerSocket
+
+import org.apache.hive.service.cli.HiveSQLException
+import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup
+import org.apache.thrift.TProcessorFactory
+import org.apache.thrift.protocol.{TBinaryProtocol, TProtocol}
+import org.apache.thrift.server.{ServerContext, TServer, TServerEventHandler, TThreadPoolServer}
+import org.apache.thrift.transport.{TServerSocket, TSSLTransportFactory, TTransport, TTransportFactory}
+
+import org.apache.livy.LivyConf
+import org.apache.livy.thriftserver.LivyCLIService
+import org.apache.livy.thriftserver.auth.AuthFactory
+
+/**
+ * This class is ported from Hive. We cannot reuse Hive's one because we need to use the
+ * `LivyCLIService`, `LivyConf` and `AuthFacotry` instead of Hive's one.
+ */
+class ThriftBinaryCLIService(override val cliService: LivyCLIService, val oomHook: Runnable)
+ extends ThriftCLIService(cliService, classOf[ThriftBinaryCLIService].getSimpleName) {
+
+ protected var server: TServer = _
+
+ override lazy val hiveAuthFactory = new AuthFactory(livyConf)
+
+ protected def initServer(): Unit = {
+ try {
+ // Server thread pool
+ val executorService = new ThreadPoolExecutorWithOomHook(
+ minWorkerThreads,
+ maxWorkerThreads,
+ workerKeepAliveTime,
+ TimeUnit.SECONDS,
+ new SynchronousQueue[Runnable],
+ new ThreadFactoryWithGarbageCleanup("LivyThriftserver-Handler-Pool"),
+ oomHook)
+ // Thrift configs
+ val transportFactory: TTransportFactory = hiveAuthFactory.getAuthTransFactory
+ val processorFactory: TProcessorFactory = hiveAuthFactory.getAuthProcFactory(this)
+ var serverSocket: TServerSocket = null
+ val serverAddress = if (hiveHost == null || hiveHost.isEmpty) {
+ new InetSocketAddress(portNum) // Wildcard bind
+ } else {
+ new InetSocketAddress(hiveHost, portNum)
+ }
+ if (!livyConf.getBoolean(LivyConf.THRIFT_USE_SSL)) {
+ serverSocket = new TServerSocket(serverAddress)
+ } else {
+ val sslVersionBlacklist = new util.ArrayList[String]
+ livyConf.get(LivyConf.THRIFT_SSL_PROTOCOL_BLACKLIST).split(",").foreach { sslVersion =>
+ sslVersionBlacklist.add(sslVersion.trim.toLowerCase)
+ }
+ val keyStorePath = livyConf.get(LivyConf.SSL_KEYSTORE).trim
+ if (keyStorePath.isEmpty) {
+ throw new IllegalArgumentException(
+ s"${LivyConf.SSL_KEYSTORE.key} Not configured for SSL connection")
+ }
+ val keyStorePassword = livyConf.get(LivyConf.SSL_KEYSTORE_PASSWORD)
+ val params = new TSSLTransportFactory.TSSLTransportParameters
+ params.setKeyStore(keyStorePath, keyStorePassword)
+ serverSocket =
+ TSSLTransportFactory.getServerSocket(portNum, 0, serverAddress.getAddress, params)
+ if (serverSocket.getServerSocket.isInstanceOf[SSLServerSocket]) {
+ val sslServerSocket = serverSocket.getServerSocket.asInstanceOf[SSLServerSocket]
+ val enabledProtocols = sslServerSocket.getEnabledProtocols.filter { protocol =>
+ if (sslVersionBlacklist.contains(protocol.toLowerCase)) {
+ debug(s"Disabling SSL Protocol: $protocol")
+ false
+ } else {
+ true
+ }
+ }
+ sslServerSocket.setEnabledProtocols(enabledProtocols)
+ info(s"SSL Server Socket Enabled Protocols: ${sslServerSocket.getEnabledProtocols}")
+ }
+ }
+ // Server args
+ val maxMessageSize = livyConf.getInt(LivyConf.THRIFT_MAX_MESSAGE_SIZE)
+ val requestTimeout =
+ livyConf.getTimeAsMs(LivyConf.THRIFT_LOGIN_TIMEOUT).asInstanceOf[Int]
+ val beBackoffSlotLength =
+ livyConf.getTimeAsMs(LivyConf.THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH).asInstanceOf[Int]
+ val sargs = new TThreadPoolServer.Args(serverSocket)
+ .processorFactory(processorFactory)
+ .transportFactory(transportFactory)
+ .protocolFactory(new TBinaryProtocol.Factory)
+ .inputProtocolFactory(
+ new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize))
+ .requestTimeout(requestTimeout)
+ .requestTimeoutUnit(TimeUnit.MILLISECONDS)
+ .beBackoffSlotLength(beBackoffSlotLength)
+ .beBackoffSlotLengthUnit(TimeUnit.MILLISECONDS)
+ .executorService(executorService)
+ // TCP Server
+ server = new TThreadPoolServer(sargs)
+ server.setServerEventHandler(new TServerEventHandler() {
+ override def createContext(input: TProtocol, output: TProtocol): ServerContext = {
+ new ThriftCLIServerContext
+ }
+
+ override def deleteContext(
+ serverContext: ServerContext,
+ input: TProtocol,
+ output: TProtocol): Unit = {
+ val context = serverContext.asInstanceOf[ThriftCLIServerContext]
+ val sessionHandle = context.getSessionHandle
+ if (sessionHandle != null) {
+ info("Session disconnected without closing properly. ")
+ try {
+ val close = livyConf.getBoolean(LivyConf.THRIFT_CLOSE_SESSION_ON_DISCONNECT)
+ info("Closing the session: " + sessionHandle)
+ if (close) {
+ cliService.closeSession(sessionHandle)
+ }
+ } catch {
+ case e: HiveSQLException => warn("Failed to close session: " + e, e)
+ }
+ }
+ }
+
+ override def preServe(): Unit = {}
+
+ override def processContext(
+ serverContext: ServerContext,
+ input: TTransport,
+ output: TTransport): Unit = {
+ currentServerContext.set(serverContext)
+ }
+ })
+ info(s"Starting ${classOf[ThriftBinaryCLIService].getSimpleName} on port $portNum " +
+ s"with $minWorkerThreads...$maxWorkerThreads worker threads")
+ } catch {
+ case e: Exception => throw new RuntimeException("Failed to init thrift server", e)
+ }
+ }
+
+ override def run(): Unit = {
+ try {
+ server.serve()
+ } catch {
+ case t: InterruptedException =>
+ // This is likely a shutdown
+ info(s"Caught ${t.getClass.getSimpleName}. Shutting down thrift server.")
+ case t: Throwable =>
+ error(s"Exception caught by ${this.getClass.getSimpleName}. Exiting.", t)
+ System.exit(-1)
+ }
+ }
+
+ protected def stopServer(): Unit = {
+ server.stop()
+ server = null
+ info("Thrift server has stopped")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftCLIService.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftCLIService.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftCLIService.scala
new file mode 100644
index 0000000..4a3276f
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftCLIService.scala
@@ -0,0 +1,745 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.cli
+
+import java.io.IOException
+import java.net.{InetAddress, UnknownHostException}
+import java.util
+import java.util.Collections
+import javax.security.auth.login.LoginException
+
+import scala.collection.JavaConverters._
+
+import com.google.common.base.Preconditions.checkArgument
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.security.authentication.util.KerberosName
+import org.apache.hadoop.security.authorize.ProxyUsers
+import org.apache.hadoop.util.StringUtils
+import org.apache.hive.service.{ServiceException, ServiceUtils}
+import org.apache.hive.service.auth.{HiveAuthConstants, TSetIpAddressProcessor}
+import org.apache.hive.service.auth.HiveAuthConstants.AuthTypes
+import org.apache.hive.service.cli._
+import org.apache.hive.service.rpc.thrift._
+import org.apache.thrift.TException
+import org.apache.thrift.server.ServerContext
+
+import org.apache.livy.LivyConf
+import org.apache.livy.thriftserver.{LivyCLIService, LivyThriftServer, SessionInfo, ThriftService}
+import org.apache.livy.thriftserver.auth.AuthFactory
+
+/**
+ * This class is ported from Hive. We cannot reuse Hive's one because we need to use the
+ * `LivyCLIService`, `LivyConf` and `AuthFacotry` instead of Hive's one.
+ */
+abstract class ThriftCLIService(val cliService: LivyCLIService, val serviceName: String)
+ extends ThriftService(serviceName) with TCLIService.Iface with Runnable {
+
+ def hiveAuthFactory: AuthFactory
+
+ protected val currentServerContext = new ThreadLocal[ServerContext]
+ protected var portNum: Int = 0
+ protected var serverIPAddress: InetAddress = _
+ protected var hiveHost: String = _
+ private var isStarted: Boolean = false
+ protected var isEmbedded: Boolean = false
+ protected var livyConf: LivyConf = _
+ protected var minWorkerThreads: Int = 0
+ protected var maxWorkerThreads: Int = 0
+ protected var workerKeepAliveTime: Long = 0L
+ private var serverThread: Thread = _
+
+ override def init(conf: LivyConf): Unit = {
+ livyConf = conf
+ hiveHost = livyConf.get(LivyConf.THRIFT_BIND_HOST)
+ try {
+ if (hiveHost == null || hiveHost.isEmpty) {
+ serverIPAddress = InetAddress.getLocalHost
+ } else {
+ serverIPAddress = InetAddress.getByName(hiveHost)
+ }
+ } catch {
+ case e: UnknownHostException =>
+ throw new ServiceException(e)
+ }
+ portNum = livyConf.getInt(LivyConf.THRIFT_SERVER_PORT)
+ workerKeepAliveTime = livyConf.getTimeAsMs(LivyConf.THRIFT_WORKER_KEEPALIVE_TIME) / 1000
+ minWorkerThreads = livyConf.getInt(LivyConf.THRIFT_MIN_WORKER_THREADS)
+ maxWorkerThreads = livyConf.getInt(LivyConf.THRIFT_MAX_WORKER_THREADS)
+ super.init(livyConf)
+ }
+
+ protected def initServer(): Unit
+
+ override def start(): Unit = {
+ super.start()
+ if (!isStarted && !isEmbedded) {
+ initServer()
+ serverThread = new Thread(this)
+ serverThread.setName("Thrift Server")
+ serverThread.start()
+ isStarted = true
+ }
+ }
+
+ protected def stopServer(): Unit
+
+ override def stop(): Unit = {
+ if (isStarted && !isEmbedded) {
+ if (serverThread != null) {
+ serverThread.interrupt()
+ serverThread = null
+ }
+ stopServer()
+ isStarted = false
+ }
+ super.stop()
+ }
+
+ def getPortNumber: Int = portNum
+
+ def getServerIPAddress: InetAddress = serverIPAddress
+
+ @throws[TException]
+ override def GetDelegationToken(req: TGetDelegationTokenReq): TGetDelegationTokenResp = {
+ val resp: TGetDelegationTokenResp = new TGetDelegationTokenResp
+ if (!hiveAuthFactory.isSASLKerberosUser) {
+ resp.setStatus(unsecureTokenErrorStatus)
+ } else {
+ try {
+ val token = cliService.getDelegationToken(
+ new SessionHandle(req.getSessionHandle), hiveAuthFactory, req.getOwner, req.getRenewer)
+ resp.setDelegationToken(token)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: HiveSQLException =>
+ error("Error obtaining delegation token", e)
+ val tokenErrorStatus = HiveSQLException.toTStatus(e)
+ tokenErrorStatus.setSqlState("42000")
+ resp.setStatus(tokenErrorStatus)
+ }
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def CancelDelegationToken(req: TCancelDelegationTokenReq): TCancelDelegationTokenResp = {
+ val resp: TCancelDelegationTokenResp = new TCancelDelegationTokenResp
+ if (!hiveAuthFactory.isSASLKerberosUser) {
+ resp.setStatus(unsecureTokenErrorStatus)
+ } else {
+ try {
+ cliService.cancelDelegationToken(
+ new SessionHandle(req.getSessionHandle), hiveAuthFactory, req.getDelegationToken)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: HiveSQLException =>
+ error("Error canceling delegation token", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def RenewDelegationToken(req: TRenewDelegationTokenReq): TRenewDelegationTokenResp = {
+ val resp: TRenewDelegationTokenResp = new TRenewDelegationTokenResp
+ if (!hiveAuthFactory.isSASLKerberosUser) {
+ resp.setStatus(unsecureTokenErrorStatus)
+ } else {
+ try {
+ cliService.renewDelegationToken(
+ new SessionHandle(req.getSessionHandle), hiveAuthFactory, req.getDelegationToken)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: HiveSQLException =>
+ error("Error obtaining renewing token", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ }
+ resp
+ }
+
+ private def unsecureTokenErrorStatus: TStatus = {
+ val errorStatus: TStatus = new TStatus(TStatusCode.ERROR_STATUS)
+ errorStatus.setErrorMessage(
+ "Delegation token only supported over remote client with kerberos authentication")
+ errorStatus
+ }
+
+ @throws[TException]
+ override def OpenSession(req: TOpenSessionReq): TOpenSessionResp = {
+ info("Client protocol version: " + req.getClient_protocol)
+ val resp: TOpenSessionResp = new TOpenSessionResp
+ try {
+ val sessionHandle = getSessionHandle(req, resp)
+ resp.setSessionHandle(sessionHandle.toTSessionHandle)
+ val configurationMap: util.Map[String, String] = new util.HashMap[String, String]
+ // Set the updated fetch size from the server into the configuration map for the client
+ val defaultFetchSize =
+ Integer.toString(livyConf.getInt(LivyConf.THRIFT_RESULTSET_DEFAULT_FETCH_SIZE))
+ configurationMap.put(LivyConf.THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.key, defaultFetchSize)
+ resp.setConfiguration(configurationMap)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ Option(currentServerContext.get).foreach { context =>
+ context.asInstanceOf[ThriftCLIServerContext].setSessionHandle(sessionHandle)
+ }
+ } catch {
+ case e: Exception =>
+ warn("Error opening session: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def SetClientInfo(req: TSetClientInfoReq): TSetClientInfoResp = {
+ // TODO: We don't do anything for now, just log this for debugging.
+ // We may be able to make use of this later, e.g. for workload management.
+ if (req.isSetConfiguration) {
+ val sh = new SessionHandle(req.getSessionHandle)
+ val sb = new StringBuilder("Client information for ").append(sh).append(": ")
+
+ def processEntry(e: util.Map.Entry[String, String]): Unit = {
+ sb.append(e.getKey).append(" = ").append(e.getValue)
+ if ("ApplicationName" == e.getKey) {
+ cliService.setApplicationName(sh, e.getValue)
+ }
+ }
+
+ val entries = req.getConfiguration.entrySet.asScala.toSeq
+ try {
+ entries.headOption.foreach(processEntry)
+ entries.tail.foreach { e =>
+ sb.append(", ")
+ processEntry(e)
+ }
+ } catch {
+ case ex: Exception =>
+ warn("Error setting application name", ex)
+ return new TSetClientInfoResp(HiveSQLException.toTStatus(ex))
+ }
+ info(sb.toString())
+ }
+ new TSetClientInfoResp(ThriftCLIService.OK_STATUS)
+ }
+
+ private def getIpAddress: String = {
+ // Http transport mode.
+ // We set the thread local ip address, in ThriftHttpServlet.
+ val clientIpAddress = if (LivyThriftServer.isHTTPTransportMode(livyConf)) {
+ SessionInfo.getIpAddress
+ } else if (hiveAuthFactory.isSASLWithKerberizedHadoop) {
+ hiveAuthFactory.getIpAddress
+ } else {
+ // NOSASL
+ TSetIpAddressProcessor.getUserIpAddress
+ }
+ debug(s"Client's IP Address: $clientIpAddress")
+ clientIpAddress
+ }
+
+ /**
+ * Returns the effective username.
+ * 1. If livy.server.thrift.allow.user.substitution = false: the username of the connecting user
+ * 2. If livy.server.thrift.allow.user.substitution = true: the username of the end user,
+ * that the connecting user is trying to proxy for.
+ * This includes a check whether the connecting user is allowed to proxy for the end user.
+ */
+ @throws[HiveSQLException]
+ @throws[IOException]
+ private def getUserName(req: TOpenSessionReq): String = {
+ val username = if (LivyThriftServer.isHTTPTransportMode(livyConf)) {
+ Option(SessionInfo.getUserName).getOrElse(req.getUsername)
+ } else if (hiveAuthFactory.isSASLWithKerberizedHadoop) {
+ Option(hiveAuthFactory.getRemoteUser).orElse(Option(TSetIpAddressProcessor.getUserName))
+ .getOrElse(req.getUsername)
+ } else {
+ Option(TSetIpAddressProcessor.getUserName).getOrElse(req.getUsername)
+ }
+ val effectiveClientUser =
+ getProxyUser(getShortName(username), req.getConfiguration, getIpAddress)
+ debug(s"Client's username: $effectiveClientUser")
+ effectiveClientUser
+ }
+
+ @throws[IOException]
+ private def getShortName(userName: String): String = {
+ Option(userName).map { un =>
+ if (hiveAuthFactory.isSASLKerberosUser) {
+ // KerberosName.getShorName can only be used for kerberos user
+ new KerberosName(un).getShortName
+ } else {
+ val indexOfDomainMatch = ServiceUtils.indexOfDomainMatch(un)
+ if (indexOfDomainMatch <= 0) {
+ un
+ } else {
+ un.substring(0, indexOfDomainMatch)
+ }
+ }
+ }.orNull
+ }
+
+ /**
+ * Create a session handle
+ */
+ @throws[HiveSQLException]
+ @throws[LoginException]
+ @throws[IOException]
+ private[thriftserver] def getSessionHandle(
+ req: TOpenSessionReq, res: TOpenSessionResp): SessionHandle = {
+ val userName = getUserName(req)
+ val ipAddress = getIpAddress
+ val protocol = getMinVersion(LivyCLIService.SERVER_VERSION, req.getClient_protocol)
+ val sessionHandle =
+ if (livyConf.getBoolean(LivyConf.THRIFT_ENABLE_DOAS) && (userName != null)) {
+ cliService.openSessionWithImpersonation(
+ protocol, userName, req.getPassword, ipAddress, req.getConfiguration, null)
+ } else {
+ cliService.openSession(protocol, userName, req.getPassword, ipAddress, req.getConfiguration)
+ }
+ res.setServerProtocolVersion(protocol)
+ sessionHandle
+ }
+
+ @throws[HiveSQLException]
+ private def getProgressedPercentage(opHandle: OperationHandle): Double = {
+ checkArgument(OperationType.EXECUTE_STATEMENT == opHandle.getOperationType)
+ 0.0
+ }
+
+ private def getMinVersion(versions: TProtocolVersion*): TProtocolVersion = {
+ val values = TProtocolVersion.values
+ var current = values(values.length - 1).getValue
+ versions.foreach { version =>
+ if (current > version.getValue) {
+ current = version.getValue
+ }
+ }
+ val res = values.find(_.getValue == current)
+ assert(res.isDefined)
+ res.get
+ }
+
+ @throws[TException]
+ override def CloseSession(req: TCloseSessionReq): TCloseSessionResp = {
+ val resp = new TCloseSessionResp
+ try {
+ val sessionHandle = new SessionHandle(req.getSessionHandle)
+ cliService.closeSession(sessionHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ Option(currentServerContext.get).foreach { ctx =>
+ ctx.asInstanceOf[ThriftCLIServerContext].setSessionHandle(null)
+ }
+ } catch {
+ case e: Exception =>
+ warn("Error closing session: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetInfo(req: TGetInfoReq): TGetInfoResp = {
+ val resp = new TGetInfoResp
+ try {
+ val getInfoValue = cliService.getInfo(
+ new SessionHandle(req.getSessionHandle), GetInfoType.getGetInfoType(req.getInfoType))
+ resp.setInfoValue(getInfoValue.toTGetInfoValue)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting info: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def ExecuteStatement(req: TExecuteStatementReq): TExecuteStatementResp = {
+ val resp = new TExecuteStatementResp
+ try {
+ val sessionHandle = new SessionHandle(req.getSessionHandle)
+ val statement = req.getStatement
+ val confOverlay = req.getConfOverlay
+ val runAsync = req.isRunAsync
+ val queryTimeout = req.getQueryTimeout
+ val operationHandle = if (runAsync) {
+ cliService.executeStatementAsync(sessionHandle, statement, confOverlay, queryTimeout)
+ } else {
+ cliService.executeStatement(sessionHandle, statement, confOverlay, queryTimeout)
+ }
+ resp.setOperationHandle(operationHandle.toTOperationHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error executing statement: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetTypeInfo(req: TGetTypeInfoReq): TGetTypeInfoResp = {
+ val resp = new TGetTypeInfoResp
+ try {
+ val operationHandle = cliService.getTypeInfo(new SessionHandle(req.getSessionHandle))
+ resp.setOperationHandle(operationHandle.toTOperationHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting type info: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetCatalogs(req: TGetCatalogsReq): TGetCatalogsResp = {
+ val resp = new TGetCatalogsResp
+ try {
+ val opHandle = cliService.getCatalogs(new SessionHandle(req.getSessionHandle))
+ resp.setOperationHandle(opHandle.toTOperationHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting catalogs: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetSchemas(req: TGetSchemasReq): TGetSchemasResp = {
+ val resp = new TGetSchemasResp
+ try {
+ val opHandle = cliService.getSchemas(
+ new SessionHandle(req.getSessionHandle), req.getCatalogName, req.getSchemaName)
+ resp.setOperationHandle(opHandle.toTOperationHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting schemas: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetTables(req: TGetTablesReq): TGetTablesResp = {
+ val resp = new TGetTablesResp
+ try {
+ val opHandle = cliService.getTables(
+ new SessionHandle(req.getSessionHandle),
+ req.getCatalogName,
+ req.getSchemaName,
+ req.getTableName,
+ req.getTableTypes)
+ resp.setOperationHandle(opHandle.toTOperationHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting tables: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetTableTypes(req: TGetTableTypesReq): TGetTableTypesResp = {
+ val resp = new TGetTableTypesResp
+ try {
+ val opHandle = cliService.getTableTypes(new SessionHandle(req.getSessionHandle))
+ resp.setOperationHandle(opHandle.toTOperationHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting table types: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetColumns(req: TGetColumnsReq): TGetColumnsResp = {
+ val resp = new TGetColumnsResp
+ try {
+ val opHandle = cliService.getColumns(
+ new SessionHandle(req.getSessionHandle),
+ req.getCatalogName,
+ req.getSchemaName,
+ req.getTableName,
+ req.getColumnName)
+ resp.setOperationHandle(opHandle.toTOperationHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting columns: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetFunctions(req: TGetFunctionsReq): TGetFunctionsResp = {
+ val resp = new TGetFunctionsResp
+ try {
+ val opHandle = cliService.getFunctions(
+ new SessionHandle(req.getSessionHandle),
+ req.getCatalogName,
+ req.getSchemaName,
+ req.getFunctionName)
+ resp.setOperationHandle(opHandle.toTOperationHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting functions: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetOperationStatus(req: TGetOperationStatusReq): TGetOperationStatusResp = {
+ val resp = new TGetOperationStatusResp
+ val operationHandle = new OperationHandle(req.getOperationHandle)
+ try {
+ val operationStatus = cliService.getOperationStatus(operationHandle, req.isGetProgressUpdate)
+ resp.setOperationState(operationStatus.state.toTOperationState)
+ resp.setErrorMessage(operationStatus.state.getErrorMessage)
+ val opException = operationStatus.operationException
+ resp.setOperationStarted(operationStatus.operationStarted)
+ resp.setOperationCompleted(operationStatus.operationCompleted)
+ resp.setHasResultSet(operationStatus.hasResultSet)
+ val executionStatus = TJobExecutionStatus.NOT_AVAILABLE
+ resp.setProgressUpdateResponse(new TProgressUpdateResp(
+ Collections.emptyList[String],
+ Collections.emptyList[util.List[String]],
+ 0.0D,
+ executionStatus,
+ "",
+ 0L))
+ if (opException != null) {
+ resp.setSqlState(opException.getSQLState)
+ resp.setErrorCode(opException.getErrorCode)
+ if (opException.getErrorCode == 29999) {
+ resp.setErrorMessage(StringUtils.stringifyException(opException))
+ } else {
+ resp.setErrorMessage(opException.getMessage)
+ }
+ } else if (OperationType.EXECUTE_STATEMENT == operationHandle.getOperationType) {
+ resp.getProgressUpdateResponse.setProgressedPercentage(
+ getProgressedPercentage(operationHandle))
+ }
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting operation status: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def CancelOperation(req: TCancelOperationReq): TCancelOperationResp = {
+ val resp = new TCancelOperationResp
+ try {
+ cliService.cancelOperation(new OperationHandle(req.getOperationHandle))
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error cancelling operation: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def CloseOperation(req: TCloseOperationReq): TCloseOperationResp = {
+ val resp = new TCloseOperationResp
+ try {
+ cliService.closeOperation(new OperationHandle(req.getOperationHandle))
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error closing operation: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetResultSetMetadata(req: TGetResultSetMetadataReq): TGetResultSetMetadataResp = {
+ val resp = new TGetResultSetMetadataResp
+ try {
+ val schema = cliService.getResultSetMetadata(new OperationHandle(req.getOperationHandle))
+ resp.setSchema(schema.toTTableSchema)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting result set metadata: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def FetchResults(req: TFetchResultsReq): TFetchResultsResp = {
+ val resp = new TFetchResultsResp
+ try {
+ // Set fetch size
+ val maxFetchSize = livyConf.getInt(LivyConf.THRIFT_RESULTSET_MAX_FETCH_SIZE)
+ if (req.getMaxRows > maxFetchSize) {
+ req.setMaxRows(maxFetchSize)
+ }
+ val rowSet = cliService.fetchResults(
+ new OperationHandle(req.getOperationHandle),
+ FetchOrientation.getFetchOrientation(req.getOrientation),
+ req.getMaxRows,
+ FetchType.getFetchType(req.getFetchType))
+ resp.setResults(rowSet.toTRowSet)
+ resp.setHasMoreRows(false)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error fetching results: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetPrimaryKeys(req: TGetPrimaryKeysReq): TGetPrimaryKeysResp = {
+ val resp = new TGetPrimaryKeysResp
+ try {
+ val opHandle = cliService.getPrimaryKeys(
+ new SessionHandle(req.getSessionHandle),
+ req.getCatalogName,
+ req.getSchemaName,
+ req.getTableName)
+ resp.setOperationHandle(opHandle.toTOperationHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting functions: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetCrossReference(req: TGetCrossReferenceReq): TGetCrossReferenceResp = {
+ val resp = new TGetCrossReferenceResp
+ try {
+ val opHandle = cliService.getCrossReference(
+ new SessionHandle(req.getSessionHandle),
+ req.getParentCatalogName,
+ req.getParentSchemaName,
+ req.getParentTableName,
+ req.getForeignCatalogName,
+ req.getForeignSchemaName,
+ req.getForeignTableName)
+ resp.setOperationHandle(opHandle.toTOperationHandle)
+ resp.setStatus(ThriftCLIService.OK_STATUS)
+ } catch {
+ case e: Exception =>
+ warn("Error getting functions: ", e)
+ resp.setStatus(HiveSQLException.toTStatus(e))
+ }
+ resp
+ }
+
+ @throws[TException]
+ override def GetQueryId(req: TGetQueryIdReq): TGetQueryIdResp = {
+ try {
+ new TGetQueryIdResp(cliService.getQueryId(req.getOperationHandle))
+ } catch {
+ case e: HiveSQLException => throw new TException(e)
+ }
+ }
+
+ override def run(): Unit
+
+ /**
+ * If the proxy user name is provided then check privileges to substitute the user.
+ */
+ @throws[HiveSQLException]
+ private def getProxyUser(
+ realUser: String,
+ sessionConf: util.Map[String, String],
+ ipAddress: String): String = {
+ var proxyUser: String = null
+ // We set the thread local proxy username, in ThriftHttpServlet.
+ if (livyConf.get(LivyConf.THRIFT_TRANSPORT_MODE).equalsIgnoreCase("http")) {
+ proxyUser = SessionInfo.getProxyUserName
+ debug("Proxy user from query string: " + proxyUser)
+ }
+ if (proxyUser == null && sessionConf != null &&
+ sessionConf.containsKey(HiveAuthConstants.HS2_PROXY_USER)) {
+ val proxyUserFromThriftBody = sessionConf.get(HiveAuthConstants.HS2_PROXY_USER)
+ debug("Proxy user from thrift body: " + proxyUserFromThriftBody)
+ proxyUser = proxyUserFromThriftBody
+ }
+ if (proxyUser == null) return realUser
+ // check whether substitution is allowed
+ if (!livyConf.getBoolean(LivyConf.THRIFT_ALLOW_USER_SUBSTITUTION)) {
+ throw new HiveSQLException("Proxy user substitution is not allowed")
+ }
+ // If there's no authentication, then directly substitute the user
+ if (AuthTypes.NONE.toString.equalsIgnoreCase(livyConf.get(LivyConf.THRIFT_AUTHENTICATION))) {
+ return proxyUser
+ }
+ // Verify proxy user privilege of the realUser for the proxyUser
+ verifyProxyAccess(realUser, proxyUser, ipAddress)
+ debug("Verified proxy user: " + proxyUser)
+ proxyUser
+ }
+
+ @throws[HiveSQLException]
+ private def verifyProxyAccess(realUser: String, proxyUser: String, ipAddress: String): Unit = {
+ try {
+ val sessionUgi = if (UserGroupInformation.isSecurityEnabled) {
+ UserGroupInformation.createProxyUser(
+ new KerberosName(realUser).getServiceName, UserGroupInformation.getLoginUser)
+ } else {
+ UserGroupInformation.createRemoteUser(realUser)
+ }
+ if (!proxyUser.equalsIgnoreCase(realUser)) {
+ ProxyUsers.refreshSuperUserGroupsConfiguration()
+ ProxyUsers.authorize(UserGroupInformation.createProxyUser(proxyUser, sessionUgi), ipAddress)
+ }
+ } catch {
+ case e: IOException =>
+ throw new HiveSQLException(
+ s"Failed to validate proxy privilege of $realUser for $proxyUser", "08S01", e)
+ }
+ }
+}
+
+object ThriftCLIService {
+ private val OK_STATUS: TStatus = new TStatus(TStatusCode.SUCCESS_STATUS)
+}
+
+private[thriftserver] class ThriftCLIServerContext extends ServerContext {
+ private var sessionHandle: SessionHandle = _
+
+ def setSessionHandle(sessionHandle: SessionHandle): Unit = {
+ this.sessionHandle = sessionHandle
+ }
+
+ def getSessionHandle: SessionHandle = sessionHandle
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftHttpCLIService.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftHttpCLIService.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftHttpCLIService.scala
new file mode 100644
index 0000000..8a3d439
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftHttpCLIService.scala
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.cli
+
+import java.util.concurrent.SynchronousQueue
+import java.util.concurrent.TimeUnit
+import javax.ws.rs.HttpMethod
+
+import org.apache.hive.service.rpc.thrift.TCLIService
+import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup
+import org.apache.thrift.protocol.TBinaryProtocol
+import org.eclipse.jetty.server.HttpConfiguration
+import org.eclipse.jetty.server.HttpConnectionFactory
+import org.eclipse.jetty.server.Server
+import org.eclipse.jetty.server.ServerConnector
+import org.eclipse.jetty.server.handler.gzip.GzipHandler
+import org.eclipse.jetty.servlet.ServletContextHandler
+import org.eclipse.jetty.servlet.ServletHolder
+import org.eclipse.jetty.util.ssl.SslContextFactory
+import org.eclipse.jetty.util.thread.ExecutorThreadPool
+
+import org.apache.livy.LivyConf
+import org.apache.livy.thriftserver.LivyCLIService
+import org.apache.livy.thriftserver.auth.AuthFactory
+
+/**
+ * This class is ported from Hive. We cannot reuse Hive's one because we need to use the
+ * `LivyCLIService`, `LivyConf` and `AuthFacotry` instead of Hive's one.
+ */
+class ThriftHttpCLIService(
+ override val cliService: LivyCLIService,
+ val oomHook: Runnable)
+ extends ThriftCLIService(cliService, classOf[ThriftHttpCLIService].getSimpleName) {
+
+ protected var server: Server = _
+
+ override lazy val hiveAuthFactory = new AuthFactory(livyConf)
+
+ /**
+ * Configure Jetty to serve http requests. Example of a client connection URL:
+ * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
+ * e.g. http://gateway:port/livy/servlets/thrifths2/
+ */
+ protected def initServer(): Unit = {
+ try {
+ // Server thread pool
+ // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
+ val executorService = new ThreadPoolExecutorWithOomHook(
+ minWorkerThreads,
+ maxWorkerThreads,
+ workerKeepAliveTime,
+ TimeUnit.SECONDS,
+ new SynchronousQueue[Runnable],
+ new ThreadFactoryWithGarbageCleanup("LivyThriftserver-HttpHandler-Pool"),
+ oomHook)
+ val threadPool = new ExecutorThreadPool(executorService)
+ // HTTP Server
+ server = new Server(threadPool)
+ val conf = new HttpConfiguration
+ // Configure header size
+ val requestHeaderSize = livyConf.getInt(LivyConf.THRIFT_HTTP_REQUEST_HEADER_SIZE)
+ val responseHeaderSize = livyConf.getInt(LivyConf.THRIFT_HTTP_RESPONSE_HEADER_SIZE)
+ conf.setRequestHeaderSize(requestHeaderSize)
+ conf.setResponseHeaderSize(responseHeaderSize)
+ val http = new HttpConnectionFactory(conf)
+ val useSsl = livyConf.getBoolean(LivyConf.THRIFT_USE_SSL)
+ val schemeName = if (useSsl) "https" else "http"
+ // Change connector if SSL is used
+ val connector = if (useSsl) {
+ val keyStorePath = livyConf.get(LivyConf.SSL_KEYSTORE).trim
+ val keyStorePassword = livyConf.get(LivyConf.SSL_KEYSTORE_PASSWORD)
+ if (keyStorePath.isEmpty) {
+ throw new IllegalArgumentException(
+ s"${LivyConf.SSL_KEYSTORE.key} Not configured for SSL connection")
+ }
+ val sslContextFactory = new SslContextFactory
+ val excludedProtocols = livyConf.get(LivyConf.THRIFT_SSL_PROTOCOL_BLACKLIST).split(",")
+ info(s"HTTP Server SSL: adding excluded protocols: $excludedProtocols")
+ sslContextFactory.addExcludeProtocols(excludedProtocols: _*)
+ info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = " +
+ sslContextFactory.getExcludeProtocols)
+ sslContextFactory.setKeyStorePath(keyStorePath)
+ sslContextFactory.setKeyStorePassword(keyStorePassword)
+ new ServerConnector(server, sslContextFactory, http)
+ } else {
+ new ServerConnector(server, http)
+ }
+ connector.setPort(portNum)
+ // Linux: yes, Windows:no
+ connector.setReuseAddress(true)
+ val maxIdleTime = livyConf.getTimeAsMs(LivyConf.THRIFT_HTTP_MAX_IDLE_TIME).asInstanceOf[Int]
+ connector.setIdleTimeout(maxIdleTime)
+ server.addConnector(connector)
+ // Thrift configs
+ val processor = new TCLIService.Processor[TCLIService.Iface](this)
+ val protocolFactory = new TBinaryProtocol.Factory
+ // Set during the init phase of LivyThriftserver if auth mode is kerberos
+ // UGI for the livy/_HOST (kerberos) principal
+ val serviceUGI = cliService.getServiceUGI
+ // UGI for the http/_HOST (SPNego) principal
+ val httpUGI = cliService.getHttpUGI
+ val authType = livyConf.get(LivyConf.THRIFT_AUTHENTICATION)
+ val thriftHttpServlet = new ThriftHttpServlet(
+ processor,
+ protocolFactory,
+ authType,
+ serviceUGI,
+ httpUGI,
+ hiveAuthFactory,
+ livyConf)
+ // Context handler
+ val context = new ServletContextHandler(ServletContextHandler.SESSIONS)
+ context.setContextPath("/")
+ if (livyConf.getBoolean(LivyConf.THRIFT_XSRF_FILTER_ENABLED)) {
+ // Filtering does not work here currently, doing filter in ThriftHttpServlet
+ debug("XSRF filter enabled")
+ } else {
+ warn("XSRF filter disabled")
+ }
+ val httpPath = getHttpPath(livyConf.get(LivyConf.THRIFT_HTTP_PATH))
+ if (livyConf.getBoolean(LivyConf.THRIFT_XSRF_FILTER_ENABLED)) {
+ val gzipHandler = new GzipHandler
+ gzipHandler.setHandler(context)
+ gzipHandler.addIncludedMethods(HttpMethod.POST)
+ gzipHandler.addIncludedMimeTypes(ThriftHttpCLIService.APPLICATION_THRIFT)
+ server.setHandler(gzipHandler)
+ } else {
+ server.setHandler(context)
+ }
+ context.addServlet(new ServletHolder(thriftHttpServlet), httpPath)
+ // TODO: check defaults: maxTimeout, keepalive, maxBodySize,
+ // bodyRecieveDuration, etc.
+ // Finally, start the server
+ server.start()
+ info(s"Started ${classOf[ThriftHttpCLIService].getSimpleName} in $schemeName mode on port " +
+ s"$portNum path=$httpPath with $minWorkerThreads...$maxWorkerThreads worker threads")
+ } catch {
+ case e: Exception => throw new RuntimeException("Failed to init HttpServer", e)
+ }
+ }
+
+ override def run(): Unit = {
+ try {
+ server.join()
+ } catch {
+ case t: InterruptedException =>
+ // This is likely a shutdown
+ info(s"Caught ${t.getClass.getSimpleName}. Shutting down thrift server.")
+ case t: Throwable =>
+ error(s"Exception caught by ${this.getClass.getSimpleName}. Exiting.", t)
+ System.exit(-1)
+ }
+ }
+
+ /**
+ * The config parameter can be like "path", "/path", "/path/", "path/*",
+ * "/path1/path2/*" and so on. httpPath should end up as "/*", "/path/*" or
+ * "/path1/../pathN/*"
+ */
+ private def getHttpPath(httpPath: String): String = {
+ Option(httpPath) match {
+ case None | Some("") => "/*"
+ case Some(path) =>
+ val withStartingSlash = if (!path.startsWith("/")) {
+ s"/$path"
+ } else {
+ path
+ }
+ if (httpPath.endsWith("/")) {
+ s"$withStartingSlash*"
+ } else if (!httpPath.endsWith("/*")) {
+ s"$withStartingSlash/*"
+ } else {
+ withStartingSlash
+ }
+ }
+ }
+
+ protected def stopServer(): Unit = {
+ if ((server != null) && server.isStarted) {
+ try {
+ server.stop()
+ server = null
+ info("Thrift HTTP server has been stopped")
+ } catch {
+ case e: Exception =>
+ error("Error stopping HTTP server: ", e)
+ }
+ }
+ }
+}
+
+object ThriftHttpCLIService {
+ private val APPLICATION_THRIFT = "application/x-thrift"
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftHttpServlet.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftHttpServlet.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftHttpServlet.scala
new file mode 100644
index 0000000..ae83917
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThriftHttpServlet.scala
@@ -0,0 +1,503 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.cli
+
+import java.io.IOException
+import java.security.{PrivilegedExceptionAction, SecureRandom}
+import javax.servlet.ServletException
+import javax.servlet.http.{Cookie, HttpServletRequest, HttpServletResponse}
+import javax.ws.rs.core.NewCookie
+
+import scala.collection.JavaConverters._
+
+import org.apache.commons.codec.binary.{Base64, StringUtils}
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.security.authentication.util.KerberosName
+import org.apache.hive.service.CookieSigner
+import org.apache.hive.service.auth.{HiveAuthConstants, HttpAuthenticationException, HttpAuthUtils}
+import org.apache.hive.service.auth.HiveAuthConstants.AuthTypes
+import org.apache.hive.service.cli.HiveSQLException
+import org.apache.thrift.TProcessor
+import org.apache.thrift.protocol.TProtocolFactory
+import org.apache.thrift.server.TServlet
+import org.ietf.jgss.{GSSContext, GSSCredential, GSSException, GSSManager, Oid}
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.thriftserver.SessionInfo
+import org.apache.livy.thriftserver.auth.{AuthenticationProvider, AuthFactory}
+
+/**
+ * This class is a porting of the parts we use from `ThriftHttpServlet` by Hive.
+ */
+class ThriftHttpServlet(
+ processor: TProcessor,
+ protocolFactory: TProtocolFactory,
+ val authType: String,
+ val serviceUGI: UserGroupInformation,
+ val httpUGI: UserGroupInformation,
+ val authFactory: AuthFactory,
+ val livyConf: LivyConf) extends TServlet(processor, protocolFactory) with Logging {
+
+ private val isCookieAuthEnabled = livyConf.getBoolean(LivyConf.THRIFT_HTTP_COOKIE_AUTH_ENABLED)
+
+ // Class members for cookie based authentication.
+ private val signer: CookieSigner = if (isCookieAuthEnabled) {
+ // Generate the signer with secret.
+ val secret = ThriftHttpServlet.RAN.nextLong.toString
+ debug("Using the random number as the secret for cookie generation " + secret)
+ new CookieSigner(secret.getBytes())
+ } else {
+ null
+ }
+
+ private val cookieDomain = livyConf.get(LivyConf.THRIFT_HTTP_COOKIE_DOMAIN)
+ private val cookiePath = livyConf.get(LivyConf.THRIFT_HTTP_COOKIE_PATH)
+ private val cookieMaxAge =
+ (livyConf.getTimeAsMs(LivyConf.THRIFT_HTTP_COOKIE_MAX_AGE) / 1000).toInt
+ private val isCookieSecure = livyConf.getBoolean(LivyConf.THRIFT_USE_SSL)
+ private val isHttpOnlyCookie = livyConf.getBoolean(LivyConf.THRIFT_HTTP_COOKIE_IS_HTTPONLY)
+ private val xsrfFilterEnabled = livyConf.getBoolean(LivyConf.THRIFT_XSRF_FILTER_ENABLED)
+
+ @throws[IOException]
+ @throws[ServletException]
+ override protected def doPost(
+ request: HttpServletRequest, response: HttpServletResponse): Unit = {
+ var clientUserName: String = null
+ var requireNewCookie: Boolean = false
+
+ try {
+ if (xsrfFilterEnabled) {
+ val continueProcessing = ThriftHttpServlet.doXsrfFilter(request, response)
+ if (!continueProcessing) {
+ warn("Request did not have valid XSRF header, rejecting.")
+ return
+ }
+ }
+ // If the cookie based authentication is already enabled, parse the
+ // request and validate the request cookies.
+ if (isCookieAuthEnabled) {
+ clientUserName = validateCookie(request)
+ requireNewCookie = clientUserName == null
+ if (requireNewCookie) {
+ info("Could not validate cookie sent, will try to generate a new cookie")
+ }
+ }
+ // If the cookie based authentication is not enabled or the request does
+ // not have a valid cookie, use the kerberos or password based authentication
+ // depending on the server setup.
+ if (clientUserName == null) {
+ // For a kerberos setup
+ if (ThriftHttpServlet.isKerberosAuthMode(authType)) {
+ val delegationToken = request.getHeader(ThriftHttpServlet.HIVE_DELEGATION_TOKEN_HEADER)
+ // Each http request must have an Authorization header
+ if ((delegationToken != null) && (!delegationToken.isEmpty)) {
+ clientUserName = doTokenAuth(request, response)
+ } else {
+ clientUserName = doKerberosAuth(request)
+ }
+ } else {
+ // For password based authentication
+ clientUserName = doPasswdAuth(request, authType)
+ }
+ }
+ debug(s"Client username: $clientUserName")
+
+ // Set the thread local username to be used for doAs if true
+ SessionInfo.setUserName(clientUserName)
+
+ // find proxy user if any from query param
+ val doAsQueryParam = ThriftHttpServlet.getDoAsQueryParam(request.getQueryString)
+ if (doAsQueryParam != null) {
+ SessionInfo.setProxyUserName(doAsQueryParam)
+ }
+
+ val clientIpAddress = request.getRemoteAddr
+ debug("Client IP Address: " + clientIpAddress)
+ // Set the thread local ip address
+ SessionInfo.setIpAddress(clientIpAddress)
+
+ // get forwarded hosts address
+ val forwardedFor = request.getHeader(ThriftHttpServlet.X_FORWARDED_FOR)
+ if (forwardedFor != null) {
+ debug(s"${ThriftHttpServlet.X_FORWARDED_FOR}:$forwardedFor")
+ SessionInfo.setForwardedAddresses(forwardedFor.split(",").toList.asJava)
+ } else {
+ SessionInfo.setForwardedAddresses(List.empty.asJava)
+ }
+
+ // Generate new cookie and add it to the response
+ if (requireNewCookie && !authType.equalsIgnoreCase(AuthTypes.NOSASL.toString)) {
+ val cookieToken = HttpAuthUtils.createCookieToken(clientUserName)
+ val hs2Cookie = createCookie(signer.signCookie(cookieToken))
+
+ if (isHttpOnlyCookie) {
+ response.setHeader("SET-COOKIE", ThriftHttpServlet.getHttpOnlyCookieHeader(hs2Cookie))
+ } else {
+ response.addCookie(hs2Cookie)
+ }
+ info("Cookie added for clientUserName " + clientUserName)
+ }
+ super.doPost(request, response);
+ } catch {
+ case e: HttpAuthenticationException =>
+ error("Error: ", e)
+ // Send a 401 to the client
+ response.setStatus(HttpServletResponse.SC_UNAUTHORIZED)
+ if(ThriftHttpServlet.isKerberosAuthMode(authType)) {
+ response.addHeader(HttpAuthUtils.WWW_AUTHENTICATE, HttpAuthUtils.NEGOTIATE)
+ }
+ // scalastyle:off println
+ response.getWriter.println("Authentication Error: " + e.getMessage)
+ // scalastyle:on println
+ } finally {
+ // Clear the thread locals
+ SessionInfo.clearUserName()
+ SessionInfo.clearIpAddress()
+ SessionInfo.clearProxyUserName()
+ SessionInfo.clearForwardedAddresses()
+ }
+ }
+
+ /**
+ * Retrieves the client name from cookieString. If the cookie does not correspond to a valid
+ * client, the function returns null.
+ * @param cookies HTTP Request cookies.
+ * @return Client Username if cookieString has a HS2 Generated cookie that is currently valid.
+ * Else, returns null.
+ */
+ private def getClientNameFromCookie(cookies: Array[Cookie]): String = {
+ // Following is the main loop which iterates through all the cookies send by the client.
+ // The HS2 generated cookies are of the format hive.server2.auth=<value>
+ // A cookie which is identified as a hiveserver2 generated cookie is validated by calling
+ // signer.verifyAndExtract(). If the validation passes, send the username for which the cookie
+ // is validated to the caller. If no client side cookie passes the validation, return null to
+ // the caller.
+ cookies.filter(_.equals(ThriftHttpServlet.AUTH_COOKIE)).foreach { cookie =>
+ val value = signer.verifyAndExtract(cookie.getValue)
+ if (value != null) {
+ val userName = HttpAuthUtils.getUserNameFromCookieToken(value)
+ if (userName == null) {
+ warn("Invalid cookie token " + value)
+ } else {
+ // We have found a valid cookie in the client request.
+ if (logger.isDebugEnabled()) {
+ debug("Validated the cookie for user " + userName)
+ }
+ return userName
+ }
+ }
+ }
+ // No valid generated cookies found, return null
+ null
+ }
+
+ /**
+ * Convert cookie array to human readable cookie string
+ * @param cookies Cookie Array
+ * @return String containing all the cookies separated by a newline character.
+ * Each cookie is of the format [key]=[value]
+ */
+ private def toCookieStr(cookies: Array[Cookie]): String = {
+ cookies.map(c => s"${c.getName} = ${c.getValue} ;\n").mkString
+ }
+
+ /**
+ * Validate the request cookie. This function iterates over the request cookie headers
+ * and finds a cookie that represents a valid client/server session. If it finds one, it
+ * returns the client name associated with the session. Else, it returns null.
+ * @param request The HTTP Servlet Request send by the client
+ * @return Client Username if the request has valid HS2 cookie, else returns null
+ */
+ private def validateCookie(request: HttpServletRequest): String = {
+ // Find all the valid cookies associated with the request.
+ val cookies = request.getCookies
+
+ if (cookies == null) {
+ if (logger.isDebugEnabled()) {
+ debug("No valid cookies associated with the request " + request)
+ }
+ null
+ } else {
+ if (logger.isDebugEnabled()) {
+ debug("Received cookies: " + toCookieStr(cookies))
+ }
+ getClientNameFromCookie(cookies)
+ }
+ }
+
+ /**
+ * Generate a server side cookie given the cookie value as the input.
+ * @param str Input string token.
+ * @return The generated cookie.
+ */
+ private def createCookie(str: String): Cookie = {
+ if (logger.isDebugEnabled()) {
+ debug(s"Cookie name = ${ThriftHttpServlet.AUTH_COOKIE} value = $str")
+ }
+ val cookie = new Cookie(ThriftHttpServlet.AUTH_COOKIE, str)
+
+ cookie.setMaxAge(cookieMaxAge)
+ if (cookieDomain != null) {
+ cookie.setDomain(cookieDomain)
+ }
+ if (cookiePath != null) {
+ cookie.setPath(cookiePath)
+ }
+ cookie.setSecure(isCookieSecure)
+ cookie
+ }
+
+
+ /**
+ * Do the authentication (LDAP/PAM not yet supported)
+ */
+ private def doPasswdAuth(request: HttpServletRequest, authType: String): String = {
+ val userName = getUsername(request, authType)
+ // No-op when authType is NOSASL
+ if (!authType.equalsIgnoreCase(HiveAuthConstants.AuthTypes.NOSASL.toString)) {
+ try {
+ val provider = AuthenticationProvider.getAuthenticationProvider(authType, livyConf)
+ provider.Authenticate(userName, getPassword(request, authType))
+ } catch {
+ case e: Exception => throw new HttpAuthenticationException(e)
+ }
+ }
+ userName
+ }
+
+ private def doTokenAuth(request: HttpServletRequest, response: HttpServletResponse): String = {
+ val tokenStr = request.getHeader(ThriftHttpServlet.HIVE_DELEGATION_TOKEN_HEADER)
+ try {
+ authFactory.verifyDelegationToken(tokenStr)
+ } catch {
+ case e: HiveSQLException => throw new HttpAuthenticationException(e);
+ }
+ }
+
+ /**
+ * Do the GSS-API kerberos authentication. We already have a logged in subject in the form of
+ * serviceUGI, which GSS-API will extract information from.
+ * In case of a SPNego request we use the httpUGI, for the authenticating service tickets.
+ */
+ private def doKerberosAuth(request: HttpServletRequest): String = {
+ // Try authenticating with the http/_HOST principal
+ if (httpUGI != null) {
+ try {
+ return httpUGI.doAs(new HttpKerberosServerAction(request, httpUGI, authType))
+ } catch {
+ case _: Exception =>
+ info("Failed to authenticate with http/_HOST kerberos principal, trying with " +
+ "livy/_HOST kerberos principal")
+ }
+ }
+ // Now try with livy/_HOST principal
+ try {
+ serviceUGI.doAs(new HttpKerberosServerAction(request, serviceUGI, authType))
+ } catch {
+ case e: Exception =>
+ error("Failed to authenticate with livy/_HOST kerberos principal")
+ throw new HttpAuthenticationException(e)
+ }
+ }
+
+ private def getUsername(request: HttpServletRequest, authType: String): String = {
+ val creds = getAuthHeaderTokens(request, authType)
+ // Username must be present
+ if (creds(0) == null || creds(0).isEmpty) {
+ throw new HttpAuthenticationException("Authorization header received " +
+ "from the client does not contain username.")
+ }
+ creds(0)
+ }
+
+ private def getPassword(request: HttpServletRequest, authType: String): String = {
+ val creds = getAuthHeaderTokens(request, authType)
+ // Password must be present
+ if (creds(1) == null || creds(1).isEmpty) {
+ throw new HttpAuthenticationException("Authorization header received " +
+ "from the client does not contain password.")
+ }
+ creds(1)
+ }
+
+ private def getAuthHeaderTokens(request: HttpServletRequest, authType: String): Array[String] = {
+ val authHeaderBase64 = ThriftHttpServlet.getAuthHeader(request, authType)
+ val authHeaderString = StringUtils.newStringUtf8(
+ Base64.decodeBase64(authHeaderBase64.getBytes()))
+ authHeaderString.split(":")
+ }
+}
+
+
+object ThriftHttpServlet extends Logging {
+ private val XSRF_HEADER_DEFAULT = "X-XSRF-HEADER"
+ private val XSRF_METHODS_TO_IGNORE_DEFAULT = Set("GET", "OPTIONS", "HEAD", "TRACE")
+ val AUTH_COOKIE = "hive.server2.auth"
+ val RAN: SecureRandom = new SecureRandom()
+ val HIVE_DELEGATION_TOKEN_HEADER: String = "X-Hive-Delegation-Token"
+ val X_FORWARDED_FOR: String = "X-Forwarded-For"
+
+ /**
+ * Generate httponly cookie from HS2 cookie
+ * @param cookie HS2 generated cookie
+ * @return The httponly cookie
+ */
+ private def getHttpOnlyCookieHeader(cookie: Cookie): String = {
+ val newCookie = new NewCookie(
+ cookie.getName,
+ cookie.getValue,
+ cookie.getPath,
+ cookie.getDomain,
+ cookie.getVersion,
+ cookie.getComment,
+ cookie.getMaxAge,
+ cookie.getSecure)
+ newCookie + "; HttpOnly"
+ }
+
+ private def getDoAsQueryParam(queryString: String): String = {
+ if (logger.isDebugEnabled()) {
+ debug("URL query string:" + queryString)
+ }
+ Option(queryString).flatMap { qs =>
+ val params = javax.servlet.http.HttpUtils.parseQueryString(qs)
+ params.keySet().asScala.find(_.equalsIgnoreCase("doAs")).map(params.get(_).head)
+ }.orNull
+ }
+
+ private def doXsrfFilter(request: HttpServletRequest, response: HttpServletResponse): Boolean = {
+ if (XSRF_METHODS_TO_IGNORE_DEFAULT.contains(request.getMethod) ||
+ request.getHeader(XSRF_HEADER_DEFAULT) != null) {
+ true
+ } else {
+ response.sendError(
+ HttpServletResponse.SC_BAD_REQUEST,
+ "Missing Required Header for Vulnerability Protection")
+ // scalastyle:off println
+ response.getWriter.println(
+ "XSRF filter denial, requests must contain header : " + XSRF_HEADER_DEFAULT)
+ // scalastyle:on println
+ false
+ }
+ }
+
+ /**
+ * Returns the base64 encoded auth header payload
+ */
+ @throws[HttpAuthenticationException]
+ private[cli] def getAuthHeader(request: HttpServletRequest, authType: String): String = {
+ val authHeader = request.getHeader(HttpAuthUtils.AUTHORIZATION)
+ // Each http request must have an Authorization header
+ if (authHeader == null || authHeader.isEmpty) {
+ throw new HttpAuthenticationException("Authorization header received " +
+ "from the client is empty.")
+ }
+
+ val beginIndex = if (isKerberosAuthMode(authType)) {
+ (HttpAuthUtils.NEGOTIATE + " ").length()
+ } else {
+ (HttpAuthUtils.BASIC + " ").length()
+ }
+ val authHeaderBase64String = authHeader.substring(beginIndex)
+ // Authorization header must have a payload
+ if (authHeaderBase64String == null || authHeaderBase64String.isEmpty) {
+ throw new HttpAuthenticationException("Authorization header received " +
+ "from the client does not contain any data.")
+ }
+ authHeaderBase64String
+ }
+
+ private def isKerberosAuthMode(authType: String): Boolean = {
+ authType.equalsIgnoreCase(AuthTypes.KERBEROS.toString)
+ }
+}
+
+class HttpKerberosServerAction(
+ val request: HttpServletRequest,
+ val serviceUGI: UserGroupInformation,
+ authType: String) extends PrivilegedExceptionAction[String] {
+
+ @throws[HttpAuthenticationException]
+ override def run(): String = {
+ // Get own Kerberos credentials for accepting connection
+ val manager = GSSManager.getInstance()
+ var gssContext: Option[GSSContext] = None
+ val serverPrincipal = getPrincipalWithoutRealm(serviceUGI.getUserName)
+ try {
+ // This Oid for Kerberos GSS-API mechanism.
+ val kerberosMechOid = new Oid("1.2.840.113554.1.2.2")
+ // Oid for SPNego GSS-API mechanism.
+ val spnegoMechOid = new Oid("1.3.6.1.5.5.2")
+ // Oid for kerberos principal name
+ val krb5PrincipalOid = new Oid("1.2.840.113554.1.2.2.1")
+
+ // GSS name for server
+ val serverName = manager.createName(serverPrincipal, krb5PrincipalOid)
+
+ // GSS credentials for server
+ val serverCreds = manager.createCredential(serverName,
+ GSSCredential.DEFAULT_LIFETIME,
+ Array[Oid](kerberosMechOid, spnegoMechOid),
+ GSSCredential.ACCEPT_ONLY)
+
+ // Create a GSS context
+ gssContext = Some(manager.createContext(serverCreds))
+ // Get service ticket from the authorization header
+ val serviceTicketBase64 = ThriftHttpServlet.getAuthHeader(request, authType)
+ val inToken = Base64.decodeBase64(serviceTicketBase64.getBytes())
+ gssContext.get.acceptSecContext(inToken, 0, inToken.length)
+ // Authenticate or deny based on its context completion
+ if (!gssContext.get.isEstablished) {
+ throw new HttpAuthenticationException("Kerberos authentication failed: " +
+ "unable to establish context with the service ticket provided by the client.")
+ } else {
+ getPrincipalWithoutRealmAndHost(gssContext.get.getSrcName.toString)
+ }
+ } catch {
+ case e: GSSException =>
+ throw new HttpAuthenticationException("Kerberos authentication failed: ", e)
+ } finally {
+ gssContext.foreach { ctx =>
+ try {
+ ctx.dispose()
+ } catch {
+ case _: GSSException => // No-op
+ }
+ }
+ }
+ }
+
+ private def getPrincipalWithoutRealm(fullPrincipal: String): String = {
+ val fullKerberosName = new KerberosName(fullPrincipal)
+ val serviceName = fullKerberosName.getServiceName
+ val hostName = fullKerberosName.getHostName
+ if (hostName != null) {
+ serviceName + "/" + hostName
+ } else {
+ serviceName
+ }
+ }
+
+ private def getPrincipalWithoutRealmAndHost(fullPrincipal: String): String = {
+ try {
+ new KerberosName(fullPrincipal).getShortName
+ } catch {
+ case e: IOException => throw new HttpAuthenticationException(e)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetCatalogsOperation.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetCatalogsOperation.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetCatalogsOperation.scala
new file mode 100644
index 0000000..57687b0
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetCatalogsOperation.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.operation
+
+import org.apache.hive.service.cli._
+
+import org.apache.livy.Logging
+import org.apache.livy.thriftserver.serde.ThriftResultSet
+import org.apache.livy.thriftserver.types.{BasicDataType, Field, Schema}
+
+class GetCatalogsOperation(sessionHandle: SessionHandle)
+ extends MetadataOperation(sessionHandle, OperationType.GET_TABLE_TYPES) with Logging {
+
+ protected val rowSet = ThriftResultSet.apply(GetCatalogsOperation.SCHEMA, protocolVersion)
+
+ info("Starting GetCatalogsOperation")
+
+ @throws(classOf[HiveSQLException])
+ override def runInternal(): Unit = {
+ setState(OperationState.RUNNING)
+ info("Fetching table type metadata")
+ try {
+ // catalogs are actually not supported in spark, so this is a no-op
+ setState(OperationState.FINISHED)
+ info("Fetching table type metadata has been successfully finished")
+ } catch {
+ case e: Throwable =>
+ setState(OperationState.ERROR)
+ throw new HiveSQLException(e)
+ }
+ }
+
+ @throws(classOf[HiveSQLException])
+ override def getResultSetSchema: Schema = {
+ assertState(Seq(OperationState.FINISHED))
+ GetCatalogsOperation.SCHEMA
+ }
+}
+
+object GetCatalogsOperation {
+ val SCHEMA = Schema(Field("TABLE_CAT", BasicDataType("string"),
+ "Catalog name. NULL if not applicable."))
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetTableTypesOperation.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetTableTypesOperation.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetTableTypesOperation.scala
new file mode 100644
index 0000000..aff7ace
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetTableTypesOperation.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.operation
+
+import org.apache.hive.service.cli._
+
+import org.apache.livy.Logging
+import org.apache.livy.thriftserver.serde.ThriftResultSet
+import org.apache.livy.thriftserver.types.{BasicDataType, Field, Schema}
+
+class GetTableTypesOperation(sessionHandle: SessionHandle)
+ extends MetadataOperation(sessionHandle, OperationType.GET_TABLE_TYPES) with Logging {
+
+ protected val rowSet = ThriftResultSet.apply(GetTableTypesOperation.SCHEMA, protocolVersion)
+
+ info("Starting GetTableTypesOperation")
+
+ @throws(classOf[HiveSQLException])
+ override def runInternal(): Unit = {
+ setState(OperationState.RUNNING)
+ info("Fetching table type metadata")
+ try {
+ Seq("TABLE", "VIEW").foreach { t =>
+ rowSet.addRow(Array(t))
+ }
+ setState(OperationState.FINISHED)
+ info("Fetching table type metadata has been successfully finished")
+ } catch {
+ case e: Throwable =>
+ setState(OperationState.ERROR)
+ throw new HiveSQLException(e)
+ }
+ }
+
+ @throws(classOf[HiveSQLException])
+ override def getResultSetSchema: Schema = {
+ assertState(Seq(OperationState.FINISHED))
+ GetTableTypesOperation.SCHEMA
+ }
+}
+
+object GetTableTypesOperation {
+ val SCHEMA = Schema(Field("TABLE_TYPE", BasicDataType("string"), "Table type name."))
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetTypeInfoOperation.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetTypeInfoOperation.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetTypeInfoOperation.scala
new file mode 100644
index 0000000..a587445
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/GetTypeInfoOperation.scala
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.operation
+
+import java.sql.{DatabaseMetaData, Types}
+
+import org.apache.hive.service.cli.{HiveSQLException, OperationState, OperationType, SessionHandle}
+
+import org.apache.livy.Logging
+import org.apache.livy.thriftserver.serde.ThriftResultSet
+import org.apache.livy.thriftserver.types.{BasicDataType, Field, Schema}
+
+sealed case class TypeInfo(name: String, sqlType: Int, precision: Option[Int],
+ caseSensitive: Boolean, searchable: Short, unsignedAttribute: Boolean, numPrecRadix: Option[Int])
+
+class GetTypeInfoOperation(sessionHandle: SessionHandle)
+ extends MetadataOperation(sessionHandle, OperationType.GET_TYPE_INFO) with Logging {
+
+ info("Starting GetTypeInfoOperation")
+
+ protected val rowSet = ThriftResultSet.apply(GetTypeInfoOperation.SCHEMA, protocolVersion)
+
+ @throws(classOf[HiveSQLException])
+ override def runInternal(): Unit = {
+ setState(OperationState.RUNNING)
+ info("Fetching type info metadata")
+ try {
+ GetTypeInfoOperation.TYPES.foreach { t =>
+ val data = Array[Any](
+ t.name,
+ t.sqlType,
+ t.precision,
+ null, // LITERAL_PREFIX
+ null, // LITERAL_SUFFIX
+ null, // CREATE_PARAMS
+ DatabaseMetaData.typeNullable, // All types are nullable
+ t.caseSensitive,
+ t.unsignedAttribute,
+ false, // FIXED_PREC_SCALE
+ false, // AUTO_INCREMENT
+ null, // LOCAL_TYPE_NAME
+ 0, // MINIMUM_SCALE
+ 0, // MAXIMUM_SCALE
+ null, // SQL_DATA_TYPE
+ null, // SQL_DATETIME_SUB
+ t.numPrecRadix)
+ rowSet.addRow(data)
+ }
+ setState(OperationState.FINISHED)
+ info("Fetching type info metadata has been successfully finished")
+ } catch {
+ case e: Throwable =>
+ setState(OperationState.ERROR)
+ throw new HiveSQLException(e)
+ }
+ }
+
+ @throws(classOf[HiveSQLException])
+ override def getResultSetSchema: Schema = {
+ assertState(Seq(OperationState.FINISHED))
+ GetTypeInfoOperation.SCHEMA
+ }
+}
+
+object GetTypeInfoOperation {
+ val SCHEMA = Schema(
+ Field("TYPE_NAME", BasicDataType("string"), "Type name"),
+ Field("DATA_TYPE", BasicDataType("integer"), "SQL data type from java.sql.Types"),
+ Field("PRECISION", BasicDataType("integer"), "Maximum precision"),
+ Field("LITERAL_PREFIX", BasicDataType("string"),
+ "Prefix used to quote a literal (may be null)"),
+ Field("LITERAL_SUFFIX", BasicDataType("string"),
+ "Suffix used to quote a literal (may be null)"),
+ Field("CREATE_PARAMS", BasicDataType("string"),
+ "Parameters used in creating the type (may be null)"),
+ Field("NULLABLE", BasicDataType("short"), "Can you use NULL for this type"),
+ Field("CASE_SENSITIVE", BasicDataType("boolean"), "Is it case sensitive"),
+ Field("SEARCHABLE", BasicDataType("short"), "Can you use \"WHERE\" based on this type"),
+ Field("UNSIGNED_ATTRIBUTE", BasicDataType("boolean"), "Is it unsigned"),
+ Field("FIXED_PREC_SCALE", BasicDataType("boolean"), "Can it be a money value"),
+ Field("AUTO_INCREMENT", BasicDataType("boolean"),
+ "Can it be used for an auto-increment value"),
+ Field("LOCAL_TYPE_NAME", BasicDataType("string"),
+ "Localized version of type name (may be null)"),
+ Field("MINIMUM_SCALE", BasicDataType("short"), "Minimum scale supported"),
+ Field("MAXIMUM_SCALE", BasicDataType("short"), "Maximum scale supported"),
+ Field("SQL_DATA_TYPE", BasicDataType("integer"), "Unused"),
+ Field("SQL_DATETIME_SUB", BasicDataType("integer"), "Unused"),
+ Field("NUM_PREC_RADIX", BasicDataType("integer"), "Usually 2 or 10"))
+
+ import DatabaseMetaData._
+
+ val TYPES = Seq(
+ TypeInfo("void", Types.NULL, None, false, typePredNone.toShort, true, None),
+ TypeInfo("boolean", Types.BOOLEAN, None, false, typePredBasic.toShort, true, None),
+ TypeInfo("byte", Types.TINYINT, Some(3), false, typePredBasic.toShort, false, Some(10)),
+ TypeInfo("short", Types.SMALLINT, Some(5), false, typePredBasic.toShort, false, Some(10)),
+ TypeInfo("integer", Types.INTEGER, Some(10), false, typePredBasic.toShort, false, Some(10)),
+ TypeInfo("long", Types.BIGINT, Some(19), false, typePredBasic.toShort, false, Some(10)),
+ TypeInfo("float", Types.FLOAT, Some(7), false, typePredBasic.toShort, false, Some(10)),
+ TypeInfo("double", Types.DOUBLE, Some(15), false, typePredBasic.toShort, false, Some(10)),
+ TypeInfo("date", Types.DATE, None, false, typePredBasic.toShort, true, None),
+ TypeInfo("timestamp", Types.TIMESTAMP, None, false, typePredBasic.toShort, true, None),
+ TypeInfo("string", Types.VARCHAR, None, true, typeSearchable.toShort, true, None),
+ TypeInfo("binary", Types.BINARY, None, false, typePredBasic.toShort, true, None),
+ TypeInfo("decimal", Types.DECIMAL, Some(38), false, typePredBasic.toShort, false, Some(10)),
+ TypeInfo("array", Types.ARRAY, None, false, typePredBasic.toShort, true, None),
+ TypeInfo("map", Types.OTHER, None, false, typePredNone.toShort, true, None),
+ TypeInfo("struct", Types.STRUCT, None, false, typePredBasic.toShort, true, None),
+ TypeInfo("udt", Types.OTHER, None, false, typePredNone.toShort, true, None))
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/MetadataOperation.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/MetadataOperation.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/MetadataOperation.scala
new file mode 100644
index 0000000..4db3929
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/MetadataOperation.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.operation
+
+import org.apache.hive.service.cli.{FetchOrientation, HiveSQLException, OperationState, OperationType, SessionHandle}
+
+import org.apache.livy.thriftserver.serde.ThriftResultSet
+
+abstract class MetadataOperation(sessionHandle: SessionHandle, opType: OperationType)
+ extends Operation(sessionHandle, opType) {
+ setHasResultSet(true)
+
+ protected def rowSet: ThriftResultSet
+
+ @throws[HiveSQLException]
+ override def close(): Unit = {
+ setState(OperationState.CLOSED)
+ }
+
+ @throws[HiveSQLException]
+ override def cancel(stateAfterCancel: OperationState): Unit = {
+ throw new UnsupportedOperationException("MetadataOperation.cancel()")
+ }
+
+ @throws(classOf[HiveSQLException])
+ override def getNextRowSet(orientation: FetchOrientation, maxRows: Long): ThriftResultSet = {
+ assertState(Seq(OperationState.FINISHED))
+ validateFetchOrientation(orientation)
+ if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
+ rowSet.setRowOffset(0)
+ }
+ rowSet
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/Operation.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/Operation.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/Operation.scala
new file mode 100644
index 0000000..2a139e3
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/Operation.scala
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.operation
+
+import java.util
+import java.util.concurrent.Future
+
+import org.apache.hive.service.cli.FetchOrientation
+import org.apache.hive.service.cli.HiveSQLException
+import org.apache.hive.service.cli.OperationHandle
+import org.apache.hive.service.cli.OperationState
+import org.apache.hive.service.cli.OperationType
+import org.apache.hive.service.cli.SessionHandle
+import org.apache.hive.service.rpc.thrift.TProtocolVersion
+
+import org.apache.livy.Logging
+import org.apache.livy.thriftserver.serde.ThriftResultSet
+import org.apache.livy.thriftserver.types.Schema
+
+abstract class Operation(
+ val sessionHandle: SessionHandle,
+ val opType: OperationType) extends Logging {
+
+ @volatile private var state = OperationState.INITIALIZED
+ val opHandle = new OperationHandle(opType, sessionHandle.getProtocolVersion)
+
+ protected var resultSetPresent: Boolean = false
+ @volatile private var operationException: HiveSQLException = _
+ @volatile protected var backgroundHandle: Future[_] = _
+
+ private val beginTime = System.currentTimeMillis()
+ @volatile private var lastAccessTime = beginTime
+
+ protected var operationStart: Long = _
+ protected var operationComplete: Long = _
+
+ def getBackgroundHandle: Future[_] = backgroundHandle
+
+ protected def setBackgroundHandle(backgroundHandle: Future[_]): Unit = {
+ this.backgroundHandle = backgroundHandle
+ }
+
+ def shouldRunAsync: Boolean = false
+
+ def protocolVersion: TProtocolVersion = opHandle.getProtocolVersion
+
+ def getStatus: OperationStatus = {
+ // TODO: get and return also the task status
+ OperationStatus(state, operationStart, operationComplete, resultSetPresent, operationException)
+ }
+
+ def hasResultSet: Boolean = resultSetPresent
+
+ protected def setHasResultSet(hasResultSet: Boolean): Unit = {
+ this.resultSetPresent = hasResultSet
+ opHandle.setHasResultSet(hasResultSet)
+ }
+
+ @throws[HiveSQLException]
+ protected def setState(newState: OperationState): OperationState = {
+ state.validateTransition(newState)
+ val prevState = state
+ state = newState
+ onNewState(state, prevState)
+ this.lastAccessTime = System.currentTimeMillis()
+ state
+ }
+
+ def isTimedOut(current: Long, operationTimeout: Long): Boolean = {
+ if (operationTimeout == 0) {
+ false
+ } else if (operationTimeout > 0) {
+ // check only when it's in terminal state
+ state.isTerminal && lastAccessTime + operationTimeout <= current
+ } else {
+ lastAccessTime + (- operationTimeout) <= current
+ }
+ }
+
+ protected def setOperationException(operationException: HiveSQLException): Unit = {
+ this.operationException = operationException
+ }
+
+ protected def assertState(states: Seq[OperationState]): Unit = {
+ if (!states.contains(state)) {
+ throw new HiveSQLException(s"Expected states: $states, but found $state")
+ }
+ this.lastAccessTime = System.currentTimeMillis()
+ }
+
+ def isDone: Boolean = state.isTerminal
+
+ /**
+ * Implemented by subclass of Operation class to execute specific behaviors.
+ */
+ @throws[HiveSQLException]
+ protected def runInternal(): Unit
+
+ // As of now, run does nothing else than calling runInternal. This may change in the future as
+ // additional operation before and after running may be added (as it happens in Hive).
+ @throws[HiveSQLException]
+ def run(): Unit = runInternal()
+
+ @throws[HiveSQLException]
+ def cancel(stateAfterCancel: OperationState): Unit
+
+ @throws[HiveSQLException]
+ def close(): Unit
+
+ @throws[HiveSQLException]
+ def getResultSetSchema: Schema
+
+ @throws[HiveSQLException]
+ def getNextRowSet(orientation: FetchOrientation, maxRows: Long): ThriftResultSet
+
+ /**
+ * Verify if the given fetch orientation is part of the supported orientation types.
+ */
+ @throws[HiveSQLException]
+ protected def validateFetchOrientation(orientation: FetchOrientation): Unit = {
+ if (!Operation.DEFAULT_FETCH_ORIENTATION_SET.contains(orientation)) {
+ throw new HiveSQLException(
+ s"The fetch type $orientation is not supported for this resultset", "HY106")
+ }
+ }
+
+ def getBeginTime: Long = beginTime
+
+ protected def getState: OperationState = state
+
+ protected def onNewState(newState: OperationState, prevState: OperationState): Unit = {
+ newState match {
+ case OperationState.RUNNING =>
+ markOperationStartTime()
+ case OperationState.ERROR | OperationState.FINISHED | OperationState.CANCELED =>
+ markOperationCompletedTime()
+ case _ => // Do nothing.
+ }
+ }
+
+ def getOperationComplete: Long = operationComplete
+
+ def getOperationStart: Long = operationStart
+
+ protected def markOperationStartTime(): Unit = {
+ operationStart = System.currentTimeMillis()
+ }
+
+ protected def markOperationCompletedTime(): Unit = {
+ operationComplete = System.currentTimeMillis()
+ }
+}
+
+object Operation {
+ val DEFAULT_FETCH_ORIENTATION_SET: util.EnumSet[FetchOrientation] =
+ util.EnumSet.of(FetchOrientation.FETCH_NEXT, FetchOrientation.FETCH_FIRST)
+ val DEFAULT_FETCH_ORIENTATION = FetchOrientation.FETCH_NEXT
+}
+
+case class OperationStatus(
+ state: OperationState,
+ operationStarted: Long,
+ operationCompleted: Long,
+ hasResultSet: Boolean,
+ operationException: HiveSQLException)
[5/5] incubator-livy git commit: [LIVY-502] Remove dependency on
hive-exec
Posted by va...@apache.org.
[LIVY-502] Remove dependency on hive-exec
## What changes were proposed in this pull request?
This PR removes the dependency on `hive-exec`. Only modules of Hive which are used after this PR are `hive-service-rpc` and `hive-service`. This drastically reduces the amount of JARs needed by the thriftserver module.
The PR takes the Hive classes which we were using and adapts them when necessary (or simpify when we don't need something) in order to work in the Livy thriftserver.
Most of the classes are just migrated replacing all occurrences of `HiveConf`, `HiveSession` and other Hive specific classes. Only one class has a quite different logic than Hive's and it is `AuthFactory`, as we are using a different UGI handling from Hive (as we are not running the TS in a standalone JVM).
The functionalities we are taking from Hive are:
- the thrift protocol endpoints (the classes in the `cli` package), ie. the classes handling lower level details about the communication with the client;
- the authentication layer (classes in the `auth` package, which are used in the ones in `cli`), despite this PR leaves LDAP and PAM as out of scope as they are not trivial to be ported and not needed for a working solution. We can add them later;
- the classes in the `operation` package which are used to answer to metadata queries by the JDBC driver.
## How was this patch tested?
existing UTs + manual tests
Author: Marco Gaido <mg...@hortonworks.com>
Closes #117 from mgaido91/LIVY-502.
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/47d3ee6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/47d3ee6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/47d3ee6b
Branch: refs/heads/master
Commit: 47d3ee6b6555a63d0b871788a53aab022fff518a
Parents: 39fa887
Author: Marco Gaido <mg...@hortonworks.com>
Authored: Fri Nov 30 15:57:16 2018 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Fri Nov 30 15:57:16 2018 -0800
----------------------------------------------------------------------
server/pom.xml | 4 +
.../main/scala/org/apache/livy/LivyConf.scala | 66 ++
.../org/apache/livy/server/LivyServer.scala | 9 +-
.../livy/server/ThriftServerFactory.scala | 2 +
thriftserver/server/pom.xml | 36 +-
.../hive/service/auth/HiveAuthFactory.java | 320 -------
.../apache/hive/service/cli/ColumnBasedSet.java | 205 -----
.../hive/service/cli/JobProgressUpdate.java | 55 --
.../hive/service/cli/OperationStatus.java | 75 --
.../cli/operation/GetCatalogsOperation.java | 80 --
.../cli/operation/GetTableTypesOperation.java | 101 ---
.../cli/operation/GetTypeInfoOperation.java | 147 ----
.../cli/operation/HiveTableTypeMapping.java | 61 --
.../cli/operation/MetadataOperation.java | 133 ---
.../hive/service/cli/operation/Operation.java | 290 ------
.../cli/thrift/ThriftBinaryCLIService.java | 167 ----
.../service/cli/thrift/ThriftCLIService.java | 871 -------------------
.../cli/thrift/ThriftHttpCLIService.java | 234 -----
.../service/cli/thrift/ThriftHttpServlet.java | 592 -------------
.../apache/hive/service/server/HiveServer2.java | 316 -------
.../livy/thriftserver/LivyCLIService.scala | 147 ++--
.../LivyExecuteStatementOperation.scala | 55 +-
.../thriftserver/LivyOperationManager.scala | 54 +-
.../livy/thriftserver/LivyThriftServer.scala | 60 +-
.../thriftserver/LivyThriftSessionManager.scala | 56 +-
.../thriftserver/ThriftServerFactoryImpl.scala | 5 +
.../livy/thriftserver/ThriftService.scala | 159 ++++
.../thriftserver/auth/AuthBridgeServer.scala | 299 +++++++
.../livy/thriftserver/auth/AuthFactory.scala | 196 +++++
.../auth/AuthenticationProvider.scala | 74 ++
.../auth/LivyDelegationTokenSecretManager.scala | 81 ++
.../thriftserver/auth/PlainSaslServer.scala | 184 ++++
.../cli/ThreadPoolExecutorWithOomHook.scala | 54 ++
.../cli/ThriftBinaryCLIService.scala | 176 ++++
.../thriftserver/cli/ThriftCLIService.scala | 745 ++++++++++++++++
.../thriftserver/cli/ThriftHttpCLIService.scala | 210 +++++
.../thriftserver/cli/ThriftHttpServlet.scala | 503 +++++++++++
.../operation/GetCatalogsOperation.scala | 58 ++
.../operation/GetTableTypesOperation.scala | 59 ++
.../operation/GetTypeInfoOperation.scala | 126 +++
.../operation/MetadataOperation.scala | 49 ++
.../livy/thriftserver/operation/Operation.scala | 180 ++++
.../thriftserver/serde/ThriftResultSet.scala | 124 +++
.../livy/thriftserver/types/DataTypeUtils.scala | 89 +-
.../apache/livy/thriftserver/types/Schema.scala | 111 +++
.../thriftserver/ui/ThriftJsonServlet.scala | 2 +-
.../thriftserver/ThriftServerBaseTest.scala | 11 +-
.../livy/thriftserver/ThriftServerSuites.scala | 4 +-
.../session/RegisterSessionJob.java | 2 -
.../thriftserver/session/StatementState.java | 2 -
50 files changed, 3699 insertions(+), 3940 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 0ebc329..a26b207 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -114,6 +114,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
</exclusions>
<scope>${hadoop.scope}</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/server/src/main/scala/org/apache/livy/LivyConf.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala
index 63374f1..32b3522 100644
--- a/server/src/main/scala/org/apache/livy/LivyConf.scala
+++ b/server/src/main/scala/org/apache/livy/LivyConf.scala
@@ -98,9 +98,75 @@ object LivyConf {
val LAUNCH_KERBEROS_REFRESH_INTERVAL = Entry("livy.server.launch.kerberos.refresh-interval", "1h")
val KINIT_FAIL_THRESHOLD = Entry("livy.server.launch.kerberos.kinit-fail-threshold", 5)
+ // Thrift configurations
val THRIFT_SERVER_ENABLED = Entry("livy.server.thrift.enabled", false)
val THRIFT_INCR_COLLECT_ENABLED = Entry("livy.server.thrift.incrementalCollect", false)
val THRIFT_SESSION_CREATION_TIMEOUT = Entry("livy.server.thrift.session.creationTimeout", "10m")
+ // The following configs are the same present in Hive
+ val THRIFT_RESULTSET_DEFAULT_FETCH_SIZE =
+ Entry("livy.server.thrift.resultset.default.fetch.size", 1000)
+ val THRIFT_TRANSPORT_MODE = Entry("livy.server.thrift.transport.mode", "binary")
+ val THRIFT_SERVER_PORT = Entry("livy.server.thrift.port", 10090)
+ val THRIFT_LONG_POLLING_TIMEOUT = Entry("livy.server.thrift.long.polling.timeout", "5000ms")
+ val THRIFT_LIMIT_CONNECTIONS_PER_USER = Entry("livy.server.thrift.limit.connections.per.user", 0)
+ val THRIFT_LIMIT_CONNECTIONS_PER_IPADDRESS =
+ Entry("livy.server.thrift.limit.connections.per.ipaddress", 0)
+ val THRIFT_LIMIT_CONNECTIONS_PER_USER_IPADDRESS =
+ Entry("livy.server.thrift.limit.connections.per.user.ipaddress", 0)
+ val THRIFT_SESSION_CHECK_INTERVAL = Entry("livy.server.thrift.session.check.interval", "6h")
+ val THRIFT_CLOSE_SESSION_ON_DISCONNECT =
+ Entry("livy.server.thrift.close.session.on.disconnect", true)
+ val THRIFT_IDLE_SESSION_TIMEOUT = Entry("livy.server.thrift.idle.session.timeout", "7d")
+ val THRIFT_IDLE_OPERATION_TIMEOUT = Entry("livy.server.thrift.idle.operation.timeout", "5d")
+ val THRIFT_IDLE_SESSION_CHECK_OPERATION =
+ Entry("livy.server.thrift.idle.session.check.operation", true)
+ val THRIFT_LOG_OPERATION_ENABLED = Entry("livy.server.thrift.logging.operation.enabled", true)
+ val THRIFT_ASYNC_EXEC_THREADS = Entry("livy.server.thrift.async.exec.threads", 100)
+ val THRIFT_ASYNC_EXEC_SHUTDOWN_TIMEOUT =
+ Entry("livy.server.thrift.async.exec.shutdown.timeout", "10s")
+ val THRIFT_ASYNC_EXEC_WAIT_QUEUE_SIZE =
+ Entry("livy.server.thrift.async.exec.wait.queue.size", 100)
+ val THRIFT_ASYNC_EXEC_KEEPALIVE_TIME =
+ Entry("livy.server.thrift.async.exec.keepalive.time", "10s")
+ val THRIFT_BIND_HOST = Entry("livy.server.thrift.bind.host", null)
+ val THRIFT_WORKER_KEEPALIVE_TIME = Entry("livy.server.thrift.worker.keepalive.time", "60s")
+ val THRIFT_MIN_WORKER_THREADS = Entry("livy.server.thrift.min.worker.threads", 5)
+ val THRIFT_MAX_WORKER_THREADS = Entry("livy.server.thrift.max.worker.threads", 500)
+ val THRIFT_RESULTSET_MAX_FETCH_SIZE = Entry("livy.server.thrift.resultset.max.fetch.size", 10000)
+ val THRIFT_ALLOW_USER_SUBSTITUTION = Entry("livy.server.thrift.allow.user.substitution", true)
+ val THRIFT_AUTHENTICATION = Entry("livy.server.thrift.authentication", "NONE")
+ val THRIFT_ENABLE_DOAS = Entry("livy.server.thrift.enable.doAs", true)
+ val THRIFT_SSL_PROTOCOL_BLACKLIST =
+ Entry("livy.server.thrift.ssl.protocol.blacklist", "SSLv2,SSLv3")
+ val THRIFT_USE_SSL = Entry("livy.server.thrift.use.SSL", false)
+ val THRIFT_MAX_MESSAGE_SIZE = Entry("livy.server.thrift.max.message.size", 100 * 1024 * 1024)
+ val THRIFT_LOGIN_TIMEOUT = Entry("livy.server.thrift.login.timeout", "20s")
+ val THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH =
+ Entry("livy.server.thrift.exponential.backoff.slot.length", "100ms")
+ val THRIFT_HTTP_REQUEST_HEADER_SIZE =
+ Entry("livy.server.thrift.http.request.header.size", 6*1024)
+ val THRIFT_HTTP_RESPONSE_HEADER_SIZE =
+ Entry("livy.server.thrift.http.response.header.size", 6*1024)
+ val THRIFT_HTTP_MAX_IDLE_TIME = Entry("livy.server.thrift.http.max.idle.time", "1800s")
+ val THRIFT_XSRF_FILTER_ENABLED = Entry("livy.server.thrift.xsrf.filter.enabled", false)
+ val THRIFT_HTTP_PATH = Entry("livy.server.thrift.http.path", "cliservice")
+ val THRIFT_HTTP_COMPRESSION_ENABLED = Entry("livy.server.thrift.http.compression.enabled", true)
+ val THRIFT_HTTP_COOKIE_AUTH_ENABLED = Entry("livy.server.thrift.http.cookie.auth.enabled", true)
+ val THRIFT_HTTP_COOKIE_MAX_AGE = Entry("livy.server.thrift.http.cookie.max.age", "86400s")
+ val THRIFT_HTTP_COOKIE_DOMAIN = Entry("livy.server.thrift.http.cookie.domain", null)
+ val THRIFT_HTTP_COOKIE_PATH = Entry("livy.server.thrift.http.cookie.path", null)
+ val THRIFT_HTTP_COOKIE_IS_HTTPONLY = Entry("livy.server.thrift.http.cookie.is.httponly", true)
+ val THRIFT_CUSTOM_AUTHENTICATION_CLASS =
+ Entry("livy.server.thrift.custom.authentication.class", null)
+ val THRIFT_SASL_QOP = Entry("livy.server.thrift.sasl.qop", "auth")
+ val THRIFT_DELEGATION_KEY_UPDATE_INTERVAL =
+ Entry("livy.server.thrift.delegation.key.update-interval", "1d")
+ val THRIFT_DELEGATION_TOKEN_GC_INTERVAL =
+ Entry("livy.server.thrift.delegation.token.gc-interval", "1h")
+ val THRIFT_DELEGATION_TOKEN_MAX_LIFETIME =
+ Entry("livy.server.thrift.delegation.token.max-lifetime", "7d")
+ val THRIFT_DELEGATION_TOKEN_RENEW_INTERVAL =
+ Entry("livy.server.thrift.delegation.token.renew-interval", "1d")
/**
* Recovery mode of Livy. Possible values:
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/server/src/main/scala/org/apache/livy/server/LivyServer.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/LivyServer.scala b/server/src/main/scala/org/apache/livy/server/LivyServer.scala
index 5f3bfd1..ed5bd81 100644
--- a/server/src/main/scala/org/apache/livy/server/LivyServer.scala
+++ b/server/src/main/scala/org/apache/livy/server/LivyServer.scala
@@ -280,17 +280,18 @@ class LivyServer extends Logging {
server.start()
+ thriftServerFactory.foreach {
+ _.start(livyConf, interactiveSessionManager, sessionStore, accessManager)
+ }
+
Runtime.getRuntime().addShutdownHook(new Thread("Livy Server Shutdown") {
override def run(): Unit = {
info("Shutting down Livy server.")
server.stop()
+ thriftServerFactory.foreach(_.stop())
}
})
- thriftServerFactory.foreach {
- _.start(livyConf, interactiveSessionManager, sessionStore, accessManager)
- }
-
_serverUrl = Some(s"${server.protocol}://${server.host}:${server.port}")
sys.props("livy.server.server-url") = _serverUrl.get
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala b/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala
index 00c3c9f..2c47ed8 100644
--- a/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala
+++ b/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala
@@ -33,6 +33,8 @@ trait ThriftServerFactory {
sessionStore: SessionStore,
accessManager: AccessManager): Unit
+ def stop(): Unit
+
def getServlet(basePath: String): Servlet
def getServletMappings: Seq[String]
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/pom.xml
----------------------------------------------------------------------
diff --git a/thriftserver/server/pom.xml b/thriftserver/server/pom.xml
index e1c26db..c08c3fd 100644
--- a/thriftserver/server/pom.xml
+++ b/thriftserver/server/pom.xml
@@ -52,33 +52,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hbase</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.livy</groupId>
@@ -125,6 +98,11 @@
<artifactId>libthrift</artifactId>
<version>${libthrift.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-common</artifactId>
+ <version>2.25.1</version>
+ </dependency>
<!-- needed for compiling successfully when using JobContext -->
<dependency>
@@ -141,8 +119,8 @@
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>*</artifactId>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java b/thriftserver/server/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java
deleted file mode 100644
index f2041e3..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.auth;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.security.auth.login.LoginException;
-import javax.security.sasl.AuthenticationException;
-import javax.security.sasl.Sasl;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.conf.HiveConfUtil;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.hive.metastore.security.DBTokenStore;
-import org.apache.hadoop.hive.metastore.security.DelegationTokenStore;
-import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
-import org.apache.hadoop.hive.metastore.security.MemoryTokenStore;
-import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager;
-import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore;
-import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
-import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.ProxyUsers;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.thrift.ThriftCLIService;
-import org.apache.thrift.TProcessorFactory;
-import org.apache.thrift.transport.TSaslServerTransport;
-import org.apache.thrift.transport.TTransportException;
-import org.apache.thrift.transport.TTransportFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class helps in some aspects of authentication. It creates the proper Thrift classes for the
- * given configuration as well as helps with authenticating requests.
- */
-public class HiveAuthFactory {
- private static final Logger LOG = LoggerFactory.getLogger(HiveAuthFactory.class);
-
- private HadoopThriftAuthBridge.Server saslServer;
- private String authTypeStr;
- private final String transportMode;
- private final HiveConf conf;
- private String hadoopAuth;
- private MetastoreDelegationTokenManager delegationTokenManager = null;
-
- public HiveAuthFactory(HiveConf conf) throws TTransportException {
- this.conf = conf;
- transportMode = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE);
- authTypeStr = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION);
-
- // ShimLoader.getHadoopShims().isSecurityEnabled() will only check that
- // hadoopAuth is not simple, it does not guarantee it is kerberos
- hadoopAuth = conf.get(HADOOP_SECURITY_AUTHENTICATION, "simple");
-
- // In http mode we use NOSASL as the default auth type
- if (authTypeStr == null) {
- if ("http".equalsIgnoreCase(transportMode)) {
- authTypeStr = HiveAuthConstants.AuthTypes.NOSASL.getAuthName();
- } else {
- authTypeStr = HiveAuthConstants.AuthTypes.NONE.getAuthName();
- }
- }
- if (isSASLWithKerberizedHadoop()) {
- saslServer = new HadoopThriftAuthBridge.Server();
-
- // Start delegation token manager
- delegationTokenManager = new MetastoreDelegationTokenManager();
- try {
- Object baseHandler = null;
- String tokenStoreClass = SecurityUtils.getTokenStoreClassName(conf);
-
- if (tokenStoreClass.equals(DBTokenStore.class.getName())) {
- // IMetaStoreClient is needed to access token store if DBTokenStore is to be used. It
- // will be got via Hive.get(conf).getMSC in a thread where the DelegationTokenStore
- // is called. To avoid the cyclic reference, we pass the Hive class to DBTokenStore where
- // it is used to get a threadLocal Hive object with a synchronized MetaStoreClient using
- // Java reflection.
- // Note: there will be two HS2 life-long opened MSCs, one is stored in HS2 thread local
- // Hive object, the other is in a daemon thread spawned in DelegationTokenSecretManager
- // to remove expired tokens.
- baseHandler = Hive.class;
- }
-
- delegationTokenManager.startDelegationTokenSecretManager(conf, baseHandler,
- HadoopThriftAuthBridge.Server.ServerMode.HIVESERVER2);
- saslServer.setSecretManager(delegationTokenManager.getSecretManager());
- }
- catch (IOException e) {
- throw new TTransportException("Failed to start token manager", e);
- }
- }
- }
-
- public Map<String, String> getSaslProperties() {
- Map<String, String> saslProps = new HashMap<String, String>();
- SaslQOP saslQOP = SaslQOP.fromString(conf.getVar(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP));
- saslProps.put(Sasl.QOP, saslQOP.toString());
- saslProps.put(Sasl.SERVER_AUTH, "true");
- return saslProps;
- }
-
- public TTransportFactory getAuthTransFactory() throws LoginException {
- TTransportFactory transportFactory;
- TSaslServerTransport.Factory serverTransportFactory;
-
- if (isSASLWithKerberizedHadoop()) {
- try {
- serverTransportFactory = saslServer.createSaslServerTransportFactory(
- getSaslProperties());
- } catch (TTransportException e) {
- throw new LoginException(e.getMessage());
- }
- if (authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.KERBEROS.getAuthName())) {
- // no-op
- } else if (authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.NONE.getAuthName()) ||
- authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.LDAP.getAuthName()) ||
- authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.PAM.getAuthName()) ||
- authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.CUSTOM.getAuthName())) {
- try {
- serverTransportFactory.addServerDefinition("PLAIN",
- authTypeStr, null, new HashMap<String, String>(),
- new PlainSaslHelper.PlainServerCallbackHandler(authTypeStr));
- } catch (AuthenticationException e) {
- throw new LoginException ("Error setting callback handler" + e);
- }
- } else {
- throw new LoginException("Unsupported authentication type " + authTypeStr);
- }
- transportFactory = saslServer.wrapTransportFactory(serverTransportFactory);
- } else if (authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.NONE.getAuthName()) ||
- authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.LDAP.getAuthName()) ||
- authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.PAM.getAuthName()) ||
- authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.CUSTOM.getAuthName())) {
- transportFactory = PlainSaslHelper.getPlainTransportFactory(authTypeStr);
- } else if (authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.NOSASL.getAuthName())) {
- transportFactory = new TTransportFactory();
- } else {
- throw new LoginException("Unsupported authentication type " + authTypeStr);
- }
- return transportFactory;
- }
-
- /**
- * Returns the thrift processor factory for HiveServer2 running in binary mode
- * @param service
- * @return
- * @throws LoginException
- */
- public TProcessorFactory getAuthProcFactory(ThriftCLIService service) throws LoginException {
- if (isSASLWithKerberizedHadoop()) {
- return KerberosSaslHelper.getKerberosProcessorFactory(saslServer, service);
- } else {
- return PlainSaslHelper.getPlainProcessorFactory(service);
- }
- }
-
- public String getRemoteUser() {
- return saslServer == null ? null : saslServer.getRemoteUser();
- }
-
- public String getIpAddress() {
- if (saslServer == null || saslServer.getRemoteAddress() == null) {
- return null;
- } else {
- return saslServer.getRemoteAddress().getHostAddress();
- }
- }
-
- public String getUserAuthMechanism() {
- return saslServer == null ? null : saslServer.getUserAuthMechanism();
- }
-
- public boolean isSASLWithKerberizedHadoop() {
- return "kerberos".equalsIgnoreCase(hadoopAuth)
- && !authTypeStr.equalsIgnoreCase(HiveAuthConstants.AuthTypes.NOSASL.getAuthName());
- }
-
- public boolean isSASLKerberosUser() {
- return AuthMethod.KERBEROS.getMechanismName().equals(getUserAuthMechanism())
- || AuthMethod.TOKEN.getMechanismName().equals(getUserAuthMechanism());
- }
-
- // Perform SPNEGO login using the hadoop shim API if the configuration is available
- public static UserGroupInformation loginFromSpnegoKeytabAndReturnUGI(HiveConf hiveConf)
- throws IOException {
- String principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_PRINCIPAL);
- String keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_KEYTAB);
- if (principal.isEmpty() || keyTabFile.isEmpty()) {
- throw new IOException("HiveServer2 SPNEGO principal or keytab is not correctly configured");
- } else {
- return UserGroupInformation.loginUserFromKeytabAndReturnUGI(SecurityUtil.getServerPrincipal(principal, "0.0.0.0"), keyTabFile);
- }
- }
-
- // retrieve delegation token for the given user
- public String getDelegationToken(String owner, String renewer, String remoteAddr)
- throws HiveSQLException {
- if (delegationTokenManager == null) {
- throw new HiveSQLException(
- "Delegation token only supported over kerberos authentication", "08S01");
- }
-
- try {
- String tokenStr = delegationTokenManager.getDelegationTokenWithService(owner, renewer,
- HiveAuthConstants.HS2_CLIENT_TOKEN, remoteAddr);
- if (tokenStr == null || tokenStr.isEmpty()) {
- throw new HiveSQLException(
- "Received empty retrieving delegation token for user " + owner, "08S01");
- }
- return tokenStr;
- } catch (IOException e) {
- throw new HiveSQLException(
- "Error retrieving delegation token for user " + owner, "08S01", e);
- } catch (InterruptedException e) {
- throw new HiveSQLException("delegation token retrieval interrupted", "08S01", e);
- }
- }
-
- // cancel given delegation token
- public void cancelDelegationToken(String delegationToken) throws HiveSQLException {
- if (delegationTokenManager == null) {
- throw new HiveSQLException(
- "Delegation token only supported over kerberos authentication", "08S01");
- }
- try {
- delegationTokenManager.cancelDelegationToken(delegationToken);
- } catch (IOException e) {
- throw new HiveSQLException(
- "Error canceling delegation token " + delegationToken, "08S01", e);
- }
- }
-
- public void renewDelegationToken(String delegationToken) throws HiveSQLException {
- if (delegationTokenManager == null) {
- throw new HiveSQLException(
- "Delegation token only supported over kerberos authentication", "08S01");
- }
- try {
- delegationTokenManager.renewDelegationToken(delegationToken);
- } catch (IOException e) {
- throw new HiveSQLException(
- "Error renewing delegation token " + delegationToken, "08S01", e);
- }
- }
-
- public String verifyDelegationToken(String delegationToken) throws HiveSQLException {
- if (delegationTokenManager == null) {
- throw new HiveSQLException(
- "Delegation token only supported over kerberos authentication", "08S01");
- }
- try {
- return delegationTokenManager.verifyDelegationToken(delegationToken);
- } catch (IOException e) {
- String msg = "Error verifying delegation token " + delegationToken;
- LOG.error(msg, e);
- throw new HiveSQLException(msg, "08S01", e);
- }
- }
-
- public String getUserFromToken(String delegationToken) throws HiveSQLException {
- if (delegationTokenManager == null) {
- throw new HiveSQLException(
- "Delegation token only supported over kerberos authentication", "08S01");
- }
- try {
- return delegationTokenManager.getUserFromToken(delegationToken);
- } catch (IOException e) {
- throw new HiveSQLException(
- "Error extracting user from delegation token " + delegationToken, "08S01", e);
- }
- }
-
- public static void verifyProxyAccess(String realUser, String proxyUser, String ipAddress,
- HiveConf hiveConf) throws HiveSQLException {
- try {
- UserGroupInformation sessionUgi;
- if (UserGroupInformation.isSecurityEnabled()) {
- KerberosNameShim kerbName = ShimLoader.getHadoopShims().getKerberosNameShim(realUser);
- sessionUgi = UserGroupInformation.createProxyUser(
- kerbName.getServiceName(), UserGroupInformation.getLoginUser());
- } else {
- sessionUgi = UserGroupInformation.createRemoteUser(realUser);
- }
- if (!proxyUser.equalsIgnoreCase(realUser)) {
- ProxyUsers.refreshSuperUserGroupsConfiguration(hiveConf);
- ProxyUsers.authorize(UserGroupInformation.createProxyUser(proxyUser, sessionUgi),
- ipAddress, hiveConf);
- }
- } catch (IOException e) {
- throw new HiveSQLException(
- "Failed to validate proxy privilege of " + realUser + " for " + proxyUser, "08S01", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java
deleted file mode 100644
index 0dbd239..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.io.ByteArrayInputStream;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.serde2.thrift.ColumnBuffer;
-import org.apache.hadoop.hive.serde2.thrift.Type;
-import org.apache.hive.service.rpc.thrift.TColumn;
-import org.apache.hive.service.rpc.thrift.TRow;
-import org.apache.hive.service.rpc.thrift.TRowSet;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TCompactProtocol;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.transport.TIOStreamTransport;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * ColumnBasedSet.
- */
-public class ColumnBasedSet implements RowSet {
-
- private long startOffset;
-
- private final TypeDescriptor[] descriptors; // non-null only for writing (server-side)
- private final List<ColumnBuffer> columns;
- private byte[] blob;
- private boolean isBlobBased = false;
- public static final Logger LOG = LoggerFactory.getLogger(ColumnBasedSet.class);
-
- public ColumnBasedSet(TableSchema schema) {
- descriptors = schema.toTypeDescriptors();
- columns = new ArrayList<ColumnBuffer>();
- for (ColumnDescriptor colDesc : schema.getColumnDescriptors()) {
- columns.add(new ColumnBuffer(colDesc.getType()));
- }
- }
-
- public ColumnBasedSet(TRowSet tRowSet) throws TException {
- descriptors = null;
- columns = new ArrayList<ColumnBuffer>();
- // Use TCompactProtocol to read serialized TColumns
- if (tRowSet.isSetBinaryColumns()) {
- TProtocol protocol =
- new TCompactProtocol(new TIOStreamTransport(new ByteArrayInputStream(
- tRowSet.getBinaryColumns())));
- // Read from the stream using the protocol for each column in final schema
- for (int i = 0; i < tRowSet.getColumnCount(); i++) {
- TColumn tvalue = new TColumn();
- try {
- tvalue.read(protocol);
- } catch (TException e) {
- LOG.error(e.getMessage(), e);
- throw new TException("Error reading column value from the row set blob", e);
- }
- columns.add(new ColumnBuffer(tvalue));
- }
- }
- else {
- if (tRowSet.getColumns() != null) {
- for (TColumn tvalue : tRowSet.getColumns()) {
- columns.add(new ColumnBuffer(tvalue));
- }
- }
- }
- startOffset = tRowSet.getStartRowOffset();
- }
-
- public ColumnBasedSet(TypeDescriptor[] descriptors, List<ColumnBuffer> columns, long startOffset) {
- this.descriptors = descriptors;
- this.columns = columns;
- this.startOffset = startOffset;
- }
-
- public ColumnBasedSet(TableSchema schema, boolean isBlobBased) {
- this(schema);
- this.isBlobBased = isBlobBased;
- }
-
- @Override
- public ColumnBasedSet addRow(Object[] fields) {
- if (isBlobBased) {
- this.blob = (byte[]) fields[0];
- } else {
- for (int i = 0; i < fields.length; i++) {
- TypeDescriptor descriptor = descriptors[i];
- Object field = fields[i];
- if (field != null && descriptor.getType() == Type.DECIMAL_TYPE) {
- int scale = descriptor.getDecimalDigits();
- field = ((HiveDecimal) field).toFormatString(scale);
- }
- columns.get(i).addValue(descriptor.getType(), field);
- }
- }
- return this;
- }
-
- public List<ColumnBuffer> getColumns() {
- return columns;
- }
-
- @Override
- public int numColumns() {
- return columns.size();
- }
-
- @Override
- public int numRows() {
- return columns.isEmpty() ? 0 : columns.get(0).size();
- }
-
- @Override
- public ColumnBasedSet extractSubset(int maxRows) {
- int numRows = Math.min(numRows(), maxRows);
-
- List<ColumnBuffer> subset = new ArrayList<ColumnBuffer>();
- for (int i = 0; i < columns.size(); i++) {
- subset.add(columns.get(i).extractSubset(numRows));
- }
- ColumnBasedSet result = new ColumnBasedSet(descriptors, subset, startOffset);
- startOffset += numRows;
- return result;
- }
-
- @Override
- public long getStartOffset() {
- return startOffset;
- }
-
- @Override
- public void setStartOffset(long startOffset) {
- this.startOffset = startOffset;
- }
-
- public TRowSet toTRowSet() {
- TRowSet tRowSet = new TRowSet(startOffset, new ArrayList<TRow>());
- if (isBlobBased) {
- tRowSet.setColumns(null);
- tRowSet.setBinaryColumns(blob);
- tRowSet.setColumnCount(numColumns());
- } else {
- for (int i = 0; i < columns.size(); i++) {
- tRowSet.addToColumns(columns.get(i).toTColumn());
- }
- }
- return tRowSet;
- }
-
- @Override
- public Iterator<Object[]> iterator() {
- return new Iterator<Object[]>() {
-
- private int index;
- private final Object[] convey = new Object[numColumns()];
-
- @Override
- public boolean hasNext() {
- return index < numRows();
- }
-
- @Override
- public Object[] next() {
- for (int i = 0; i < columns.size(); i++) {
- convey[i] = columns.get(i).get(index);
- }
- index++;
- return convey;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("remove");
- }
- };
- }
-
- public Object[] fill(int index, Object[] convey) {
- for (int i = 0; i < columns.size(); i++) {
- convey[i] = columns.get(i).get(index);
- }
- return convey;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/JobProgressUpdate.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/JobProgressUpdate.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/JobProgressUpdate.java
deleted file mode 100644
index 5c3d8ce..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/JobProgressUpdate.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.cli;
-
-import org.apache.hadoop.hive.common.log.ProgressMonitor;
-
-import java.util.List;
-
-public class JobProgressUpdate {
- public final double progressedPercentage;
- public final String footerSummary;
- public final long startTimeMillis;
- private final List<String> headers;
- private final List<List<String>> rows;
- public final String status;
-
-
- public JobProgressUpdate(ProgressMonitor monitor) {
- this(monitor.headers(), monitor.rows(), monitor.footerSummary(), monitor.progressedPercentage(),
- monitor.startTime(), monitor.executionStatus());
- }
-
- private JobProgressUpdate(List<String> headers, List<List<String>> rows, String footerSummary,
- double progressedPercentage, long startTimeMillis, String status) {
- this.progressedPercentage = progressedPercentage;
- this.footerSummary = footerSummary;
- this.startTimeMillis = startTimeMillis;
- this.headers = headers;
- this.rows = rows;
- this.status = status;
- }
-
- public List<String> headers() {
- return headers;
- }
-
- public List<List<String>> rows() {
- return rows;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/OperationStatus.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/OperationStatus.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/OperationStatus.java
deleted file mode 100644
index 663be4b..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/OperationStatus.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-/**
- * OperationStatus
- *
- */
-public class OperationStatus {
-
- private final OperationState state;
- private final String taskStatus;
- private final long operationStarted;
- private final long operationCompleted;
- private final boolean hasResultSet;
- private final HiveSQLException operationException;
- private JobProgressUpdate jobProgressUpdate;
-
- public OperationStatus(OperationState state, String taskStatus, long operationStarted, long operationCompleted, boolean hasResultSet, HiveSQLException operationException) {
- this.state = state;
- this.taskStatus = taskStatus;
- this.operationStarted = operationStarted;
- this.operationCompleted = operationCompleted;
- this.hasResultSet = hasResultSet;
- this.operationException = operationException;
- }
-
- public OperationState getState() {
- return state;
- }
-
- public String getTaskStatus() {
- return taskStatus;
- }
-
- public long getOperationStarted() {
- return operationStarted;
- }
-
- public long getOperationCompleted() {
- return operationCompleted;
- }
-
- public boolean getHasResultSet() {
- return hasResultSet;
- }
-
- public HiveSQLException getOperationException() {
- return operationException;
- }
-
- public void setJobProgressUpdate(JobProgressUpdate jobProgressUpdate){
- this.jobProgressUpdate = jobProgressUpdate;
- }
-
- public JobProgressUpdate jobProgressUpdate(){
- return jobProgressUpdate;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java
deleted file mode 100644
index 02b7bf3..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.hive.service.cli.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * GetCatalogsOperation.
- *
- */
-public class GetCatalogsOperation extends MetadataOperation {
-
- private static final Logger LOG = LoggerFactory.getLogger(GetCatalogsOperation.class.getName());
-
- private static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
- .addStringColumn("TABLE_CAT", "Catalog name. NULL if not applicable.");
-
- private final RowSet rowSet;
-
- public GetCatalogsOperation(SessionHandle sessionHandle) {
- super(sessionHandle, OperationType.GET_CATALOGS);
- rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion(), false);
- LOG.info("Starting GetCatalogsOperation");
- }
-
- @Override
- public void runInternal() throws HiveSQLException {
- setState(OperationState.RUNNING);
- try {
- // catalogs are actually not supported in hive, so this is a no-op
- setState(OperationState.FINISHED);
- LOG.info("Fetching catalog metadata has been successfully finished");
- } catch (HiveSQLException e) {
- setState(OperationState.ERROR);
- throw e;
- }
- }
-
- /* (non-Javadoc)
- * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
- */
- @Override
- public TableSchema getResultSetSchema() throws HiveSQLException {
- return RESULT_SET_SCHEMA;
- }
-
- /* (non-Javadoc)
- * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
- */
- @Override
- public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
- assertState(new ArrayList<OperationState>(Arrays.asList(OperationState.FINISHED)));
- validateDefaultFetchOrientation(orientation);
- if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
- rowSet.setStartOffset(0);
- }
- return rowSet.extractSubset((int)maxRows);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java
deleted file mode 100644
index 0f3cc24..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hive.service.cli.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.livy.thriftserver.LivyThriftServer$;
-
-/**
- * GetTableTypesOperation.
- *
- */
-public class GetTableTypesOperation extends MetadataOperation {
-
- private static final Logger LOG = LoggerFactory.getLogger(GetTableTypesOperation.class.getName());
-
- protected static TableSchema RESULT_SET_SCHEMA = new TableSchema()
- .addStringColumn("TABLE_TYPE", "Table type name.");
-
- private final RowSet rowSet;
- private final TableTypeMapping tableTypeMapping;
-
- public GetTableTypesOperation(SessionHandle sessionHandle) {
- super(sessionHandle, OperationType.GET_TABLE_TYPES);
- String tableMappingStr = LivyThriftServer$.MODULE$.getInstance().get().getHiveConf()
- .getVar(HiveConf.ConfVars.HIVE_SERVER2_TABLE_TYPE_MAPPING);
- tableTypeMapping = TableTypeMappingFactory.getTableTypeMapping(tableMappingStr);
- rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion(), false);
- LOG.info("Starting GetTableTypesOperation");
- }
-
- @Override
- public void runInternal() throws HiveSQLException {
- setState(OperationState.RUNNING);
- LOG.info("Fetching table type metadata");
- try {
- for (TableType type : TableType.values()) {
- String tableType = tableTypeMapping.mapToClientType(type.toString());
- rowSet.addRow(new String[] {tableType});
- if (LOG.isDebugEnabled()) {
- String debugMessage = getDebugMessage("table type", RESULT_SET_SCHEMA);
- LOG.debug(debugMessage, tableType);
- }
- }
- if (LOG.isDebugEnabled() && rowSet.numRows() == 0) {
- LOG.debug("No table type metadata has been returned.");
- }
- setState(OperationState.FINISHED);
- LOG.info("Fetching table type metadata has been successfully finished");
- } catch (Exception e) {
- setState(OperationState.ERROR);
- throw new HiveSQLException(e);
- }
- }
-
- /* (non-Javadoc)
- * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
- */
- @Override
- public TableSchema getResultSetSchema() throws HiveSQLException {
- assertState(new ArrayList<OperationState>(Arrays.asList(OperationState.FINISHED)));
- return RESULT_SET_SCHEMA;
- }
-
- /* (non-Javadoc)
- * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
- */
- @Override
- public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
- assertState(new ArrayList<OperationState>(Arrays.asList(OperationState.FINISHED)));
- validateDefaultFetchOrientation(orientation);
- if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
- rowSet.setStartOffset(0);
- }
- return rowSet.extractSubset((int)maxRows);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java
deleted file mode 100644
index 501a2a9..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.hadoop.hive.serde2.thrift.Type;
-import org.apache.hive.service.cli.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * GetTypeInfoOperation.
- *
- */
-public class GetTypeInfoOperation extends MetadataOperation {
-
- private static final Logger LOG = LoggerFactory.getLogger(GetTypeInfoOperation.class.getName());
-
- private final static TableSchema RESULT_SET_SCHEMA = new TableSchema()
- .addPrimitiveColumn("TYPE_NAME", Type.STRING_TYPE,
- "Type name")
- .addPrimitiveColumn("DATA_TYPE", Type.INT_TYPE,
- "SQL data type from java.sql.Types")
- .addPrimitiveColumn("PRECISION", Type.INT_TYPE,
- "Maximum precision")
- .addPrimitiveColumn("LITERAL_PREFIX", Type.STRING_TYPE,
- "Prefix used to quote a literal (may be null)")
- .addPrimitiveColumn("LITERAL_SUFFIX", Type.STRING_TYPE,
- "Suffix used to quote a literal (may be null)")
- .addPrimitiveColumn("CREATE_PARAMS", Type.STRING_TYPE,
- "Parameters used in creating the type (may be null)")
- .addPrimitiveColumn("NULLABLE", Type.SMALLINT_TYPE,
- "Can you use NULL for this type")
- .addPrimitiveColumn("CASE_SENSITIVE", Type.BOOLEAN_TYPE,
- "Is it case sensitive")
- .addPrimitiveColumn("SEARCHABLE", Type.SMALLINT_TYPE,
- "Can you use \"WHERE\" based on this type")
- .addPrimitiveColumn("UNSIGNED_ATTRIBUTE", Type.BOOLEAN_TYPE,
- "Is it unsigned")
- .addPrimitiveColumn("FIXED_PREC_SCALE", Type.BOOLEAN_TYPE,
- "Can it be a money value")
- .addPrimitiveColumn("AUTO_INCREMENT", Type.BOOLEAN_TYPE,
- "Can it be used for an auto-increment value")
- .addPrimitiveColumn("LOCAL_TYPE_NAME", Type.STRING_TYPE,
- "Localized version of type name (may be null)")
- .addPrimitiveColumn("MINIMUM_SCALE", Type.SMALLINT_TYPE,
- "Minimum scale supported")
- .addPrimitiveColumn("MAXIMUM_SCALE", Type.SMALLINT_TYPE,
- "Maximum scale supported")
- .addPrimitiveColumn("SQL_DATA_TYPE", Type.INT_TYPE,
- "Unused")
- .addPrimitiveColumn("SQL_DATETIME_SUB", Type.INT_TYPE,
- "Unused")
- .addPrimitiveColumn("NUM_PREC_RADIX", Type.INT_TYPE,
- "Usually 2 or 10");
-
- private final RowSet rowSet;
-
- public GetTypeInfoOperation(SessionHandle sessionHandle) {
- super(sessionHandle, OperationType.GET_TYPE_INFO);
- rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion(), false);
- LOG.info("Starting GetTypeInfoOperation");
- }
-
- @Override
- public void runInternal() throws HiveSQLException {
- setState(OperationState.RUNNING);
- LOG.info("Fetching type info metadata");
- try {
- for (Type type : Type.values()) {
- Object[] rowData = new Object[] {
- type.getName(), // TYPE_NAME
- type.toJavaSQLType(), // DATA_TYPE
- type.getMaxPrecision(), // PRECISION
- type.getLiteralPrefix(), // LITERAL_PREFIX
- type.getLiteralSuffix(), // LITERAL_SUFFIX
- type.getCreateParams(), // CREATE_PARAMS
- type.getNullable(), // NULLABLE
- type.isCaseSensitive(), // CASE_SENSITIVE
- type.getSearchable(), // SEARCHABLE
- type.isUnsignedAttribute(), // UNSIGNED_ATTRIBUTE
- type.isFixedPrecScale(), // FIXED_PREC_SCALE
- type.isAutoIncrement(), // AUTO_INCREMENT
- type.getLocalizedName(), // LOCAL_TYPE_NAME
- type.getMinimumScale(), // MINIMUM_SCALE
- type.getMaximumScale(), // MAXIMUM_SCALE
- null, // SQL_DATA_TYPE, unused
- null, // SQL_DATETIME_SUB, unused
- type.getNumPrecRadix() //NUM_PREC_RADIX
- };
- rowSet.addRow(rowData);
- if (LOG.isDebugEnabled()) {
- String debugMessage = getDebugMessage("type info", RESULT_SET_SCHEMA);
- LOG.debug(debugMessage, rowData);
- }
- }
- if (LOG.isDebugEnabled() && rowSet.numRows() == 0) {
- LOG.debug("No type info metadata has been returned.");
- }
- setState(OperationState.FINISHED);
- LOG.info("Fetching type info metadata has been successfully finished");
- } catch (Exception e) {
- setState(OperationState.ERROR);
- throw new HiveSQLException(e);
- }
- }
-
- /* (non-Javadoc)
- * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
- */
- @Override
- public TableSchema getResultSetSchema() throws HiveSQLException {
- assertState(new ArrayList<OperationState>(Arrays.asList(OperationState.FINISHED)));
- return RESULT_SET_SCHEMA;
- }
-
- /* (non-Javadoc)
- * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
- */
- @Override
- public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
- assertState(new ArrayList<OperationState>(Arrays.asList(OperationState.FINISHED)));
- validateDefaultFetchOrientation(orientation);
- if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
- rowSet.setStartOffset(0);
- }
- return rowSet.extractSubset((int)maxRows);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java
deleted file mode 100644
index 6bf2507..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.hadoop.hive.metastore.TableType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * HiveTableTypeMapping.
- * Default table type mapping
- *
- */
-public class HiveTableTypeMapping implements TableTypeMapping {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveTableTypeMapping.class);
-
- @Override
- public String[] mapToHiveType(String clientTypeName) {
- return new String[] {mapToClientType(clientTypeName)};
- }
-
- @Override
- public String mapToClientType(String hiveTypeName) {
- try {
- TableType hiveType = TableType.valueOf(hiveTypeName.toUpperCase());
- return hiveType.name();
- } catch (IllegalArgumentException e) {
- LOG.warn("Invalid hive table type " + hiveTypeName);
- return hiveTypeName;
- }
- }
-
- @Override
- public Set<String> getTableTypeNames() {
- Set<String> typeNameSet = new HashSet<String>();
- for (TableType typeNames : TableType.values()) {
- typeNameSet.add(typeNames.name());
- }
- return typeNameSet;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java
deleted file mode 100644
index 553a283..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import org.apache.hive.service.cli.ColumnDescriptor;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.cli.TableSchema;
-
-/**
- * MetadataOperation.
- *
- */
-public abstract class MetadataOperation extends Operation {
-
- protected static final String DEFAULT_HIVE_CATALOG = "";
-
- protected MetadataOperation(SessionHandle sessionHandle, OperationType opType) {
- super(sessionHandle, opType);
- setHasResultSet(true);
- }
-
-
- /* (non-Javadoc)
- * @see org.apache.hive.service.cli.Operation#close()
- */
- @Override
- public void close() throws HiveSQLException {
- setState(OperationState.CLOSED);
- }
-
- /**
- * Convert wildchars and escape sequence from JDBC format to datanucleous/regex
- */
- protected String convertIdentifierPattern(final String pattern, boolean datanucleusFormat) {
- if (pattern == null) {
- return convertPattern("%", true);
- } else {
- return convertPattern(pattern, datanucleusFormat);
- }
- }
-
- /**
- * Convert wildchars and escape sequence of schema pattern from JDBC format to datanucleous/regex
- * The schema pattern treats empty string also as wildchar
- */
- protected String convertSchemaPattern(final String pattern) {
- if ((pattern == null) || pattern.isEmpty()) {
- return convertPattern("%", true);
- } else {
- return convertPattern(pattern, true);
- }
- }
-
- /**
- * Convert a pattern containing JDBC catalog search wildcards into
- * Java regex patterns.
- *
- * @param pattern input which may contain '%' or '_' wildcard characters, or
- * these characters escaped using {@link #getSearchStringEscape()}.
- * @return replace %/_ with regex search characters, also handle escaped
- * characters.
- *
- * The datanucleus module expects the wildchar as '*'. The columns search on the
- * other hand is done locally inside the hive code and that requires the regex wildchar
- * format '.*' This is driven by the datanucleusFormat flag.
- */
- private String convertPattern(String pattern, boolean datanucleusFormat) {
- String wStr;
- if (datanucleusFormat) {
- wStr = "*";
- } else {
- wStr = ".*";
- }
- pattern = replaceAll(pattern, "([^\\\\])%", "$1" + wStr);
- pattern = replaceAll(pattern, "\\\\%", "%");
- pattern = replaceAll(pattern, "^%", wStr);
- pattern = replaceAll(pattern, "([^\\\\])_", "$1.");
- pattern = replaceAll(pattern, "\\\\_", "_");
- pattern = replaceAll(pattern, "^_", ".");
- return pattern;
- }
-
- private String replaceAll(String input, final String pattern, final String replace) {
- while (true) {
- String replaced = input.replaceAll(pattern, replace);
- if (replaced.equals(input)) {
- return replaced;
- }
- input = replaced;
- }
- }
-
- @Override
- public void cancel(OperationState stateAfterCancel) throws HiveSQLException {
- throw new UnsupportedOperationException("MetadataOperation.cancel()");
- }
-
- protected String getDebugMessage(final String type, final TableSchema resultSetSchema) {
- StringBuilder debugMessage = new StringBuilder();
- debugMessage.append("Returning ");
- debugMessage.append(type);
- debugMessage.append(" metadata: ");
- boolean firstColumn = true;
- for (ColumnDescriptor column : resultSetSchema.getColumnDescriptors()) {
- if (!firstColumn) {
- debugMessage.append(", ");
- }
- debugMessage.append(column.getName());
- debugMessage.append("={}");
- firstColumn = false;
- }
- return debugMessage.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/Operation.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/Operation.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/Operation.java
deleted file mode 100644
index 9a3b516..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/operation/Operation.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.cli.operation;
-
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationHandle;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationStatus;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.rpc.thrift.TProtocolVersion;
-import org.apache.livy.thriftserver.LivyThriftServer$;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class Operation {
- protected final SessionHandle sessionHandle;
- private volatile OperationState state = OperationState.INITIALIZED;
- private final OperationHandle opHandle;
- public static final FetchOrientation DEFAULT_FETCH_ORIENTATION = FetchOrientation.FETCH_NEXT;
- public static final Logger LOG = LoggerFactory.getLogger(Operation.class.getName());
- protected boolean hasResultSet;
- protected volatile HiveSQLException operationException;
- protected volatile Future<?> backgroundHandle;
- private ScheduledExecutorService scheduledExecutorService;
-
- private long operationTimeout;
- private volatile long lastAccessTime;
- private final long beginTime;
-
- protected long operationStart;
- protected long operationComplete;
-
- protected static final EnumSet<FetchOrientation> DEFAULT_FETCH_ORIENTATION_SET =
- EnumSet.of(FetchOrientation.FETCH_NEXT,FetchOrientation.FETCH_FIRST);
-
-
- protected Operation(SessionHandle sessionHandle, OperationType opType) {
- this(sessionHandle, null, opType);
- }
-
- protected Operation(SessionHandle sessionHandle,
- Map<String, String> confOverlay, OperationType opType) {
- this.sessionHandle = sessionHandle;
- this.opHandle = new OperationHandle(opType, sessionHandle.getProtocolVersion());
- beginTime = System.currentTimeMillis();
- lastAccessTime = beginTime;
- HiveConf conf = LivyThriftServer$.MODULE$.getInstance().get().getHiveConf();
- operationTimeout = HiveConf.getTimeVar(conf,
- HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS);
- scheduledExecutorService = Executors.newScheduledThreadPool(1);
-
- }
-
- public Future<?> getBackgroundHandle() {
- return backgroundHandle;
- }
-
- protected void setBackgroundHandle(Future<?> backgroundHandle) {
- this.backgroundHandle = backgroundHandle;
- }
-
- public boolean shouldRunAsync() {
- return false; // Most operations cannot run asynchronously.
- }
-
- public SessionHandle getSessionHandle() {
- return sessionHandle;
- }
-
- public OperationHandle getHandle() {
- return opHandle;
- }
-
- public TProtocolVersion getProtocolVersion() {
- return opHandle.getProtocolVersion();
- }
-
- public OperationType getType() {
- return opHandle.getOperationType();
- }
-
- public OperationStatus getStatus() {
- String taskStatus = null;
- try {
- taskStatus = getTaskStatus();
- } catch (HiveSQLException sqlException) {
- LOG.error("Error getting task status for " + opHandle.toString(), sqlException);
- }
- return new OperationStatus(state, taskStatus, operationStart, operationComplete, hasResultSet, operationException);
- }
-
- public boolean hasResultSet() {
- return hasResultSet;
- }
-
- protected void setHasResultSet(boolean hasResultSet) {
- this.hasResultSet = hasResultSet;
- opHandle.setHasResultSet(hasResultSet);
- }
-
- protected final OperationState setState(OperationState newState) throws HiveSQLException {
- state.validateTransition(newState);
- OperationState prevState = state;
- this.state = newState;
- onNewState(state, prevState);
- this.lastAccessTime = System.currentTimeMillis();
- return this.state;
- }
-
- public boolean isTimedOut(long current) {
- if (operationTimeout == 0) {
- return false;
- }
- if (operationTimeout > 0) {
- // check only when it's in terminal state
- return state.isTerminal() && lastAccessTime + operationTimeout <= current;
- }
- return lastAccessTime + -operationTimeout <= current;
- }
-
- public long getLastAccessTime() {
- return lastAccessTime;
- }
-
- public long getOperationTimeout() {
- return operationTimeout;
- }
-
- public void setOperationTimeout(long operationTimeout) {
- this.operationTimeout = operationTimeout;
- }
-
- protected void setOperationException(HiveSQLException operationException) {
- this.operationException = operationException;
- }
-
- protected final void assertState(List<OperationState> states) throws HiveSQLException {
- if (!states.contains(state)) {
- throw new HiveSQLException("Expected states: " + states.toString() + ", but found "
- + this.state);
- }
- this.lastAccessTime = System.currentTimeMillis();
- }
-
- public boolean isDone() {
- return state.isTerminal();
- }
-
- /**
- * Invoked before runInternal().
- * Set up some preconditions, or configurations.
- */
- protected void beforeRun() {
- // For Livy operations, this currently does nothing
- }
-
- /**
- * Invoked after runInternal(), even if an exception is thrown in runInternal().
- * Clean up resources, which was set up in beforeRun().
- */
- protected void afterRun() {
- // For Livy operations, this currently does nothing
- }
-
- /**
- * Implemented by subclass of Operation class to execute specific behaviors.
- * @throws HiveSQLException
- */
- protected abstract void runInternal() throws HiveSQLException;
-
- public void run() throws HiveSQLException {
- beforeRun();
- try {
- runInternal();
- } finally {
- afterRun();
- }
- }
-
- public abstract void cancel(OperationState stateAfterCancel) throws HiveSQLException;
-
- public abstract void close() throws HiveSQLException;
-
- public abstract TableSchema getResultSetSchema() throws HiveSQLException;
-
- public abstract RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException;
-
- public String getTaskStatus() throws HiveSQLException {
- return null;
- }
-
- /**
- * Verify if the given fetch orientation is part of the default orientation types.
- * @param orientation
- * @throws HiveSQLException
- */
- protected void validateDefaultFetchOrientation(FetchOrientation orientation)
- throws HiveSQLException {
- validateFetchOrientation(orientation, DEFAULT_FETCH_ORIENTATION_SET);
- }
-
- /**
- * Verify if the given fetch orientation is part of the supported orientation types.
- * @param orientation
- * @param supportedOrientations
- * @throws HiveSQLException
- */
- protected void validateFetchOrientation(FetchOrientation orientation,
- EnumSet<FetchOrientation> supportedOrientations) throws HiveSQLException {
- if (!supportedOrientations.contains(orientation)) {
- throw new HiveSQLException("The fetch type " + orientation.toString() +
- " is not supported for this resultset", "HY106");
- }
- }
-
- protected HiveSQLException toSQLException(String prefix, CommandProcessorResponse response) {
- HiveSQLException ex = new HiveSQLException(prefix + ": " + response.getErrorMessage(),
- response.getSQLState(), response.getResponseCode());
- if (response.getException() != null) {
- ex.initCause(response.getException());
- }
- return ex;
- }
-
- public long getBeginTime() {
- return beginTime;
- }
-
- protected OperationState getState() {
- return state;
- }
-
- protected void onNewState(OperationState state, OperationState prevState) {
- switch(state) {
- case RUNNING:
- markOperationStartTime();
- break;
- case ERROR:
- case FINISHED:
- case CANCELED:
- markOperationCompletedTime();
- break;
- }
- }
-
- public long getOperationComplete() {
- return operationComplete;
- }
-
- public long getOperationStart() {
- return operationStart;
- }
-
- protected void markOperationStartTime() {
- operationStart = System.currentTimeMillis();
- }
-
- protected void markOperationCompletedTime() {
- operationComplete = System.currentTimeMillis();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
deleted file mode 100644
index 7873e33..0000000
--- a/thriftserver/server/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hive.common.auth.HiveAuthUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.ICLIService;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
-import org.apache.thrift.TProcessorFactory;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.server.ServerContext;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.server.TServerEventHandler;
-import org.apache.thrift.server.TThreadPoolServer;
-import org.apache.thrift.transport.TServerSocket;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportFactory;
-
-
-public class ThriftBinaryCLIService extends ThriftCLIService {
- private final Runnable oomHook;
- protected TServer server;
-
- public ThriftBinaryCLIService(ICLIService cliService, Runnable oomHook) {
- super(cliService, ThriftBinaryCLIService.class.getSimpleName());
- this.oomHook = oomHook;
- }
-
- @Override
- protected void initServer() {
- try {
- // Server thread pool
- String threadPoolName = "HiveServer2-Handler-Pool";
- ExecutorService executorService = new ThreadPoolExecutorWithOomHook(minWorkerThreads, maxWorkerThreads,
- workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
- new ThreadFactoryWithGarbageCleanup(threadPoolName), oomHook);
-
- // Thrift configs
- hiveAuthFactory = new HiveAuthFactory(hiveConf);
- TTransportFactory transportFactory = hiveAuthFactory.getAuthTransFactory();
- TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this);
- TServerSocket serverSocket = null;
- List<String> sslVersionBlacklist = new ArrayList<String>();
- for (String sslVersion : hiveConf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST).split(",")) {
- sslVersionBlacklist.add(sslVersion);
- }
- if (!hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL)) {
- serverSocket = HiveAuthUtils.getServerSocket(hiveHost, portNum);
- } else {
- String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim();
- if (keyStorePath.isEmpty()) {
- throw new IllegalArgumentException(
- ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname + " Not configured for SSL connection");
- }
- String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf,
- HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname);
- serverSocket = HiveAuthUtils.getServerSSLSocket(hiveHost, portNum, keyStorePath, keyStorePassword,
- sslVersionBlacklist);
- }
-
- // Server args
- int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE);
- int requestTimeout = (int) hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_TIMEOUT,
- TimeUnit.SECONDS);
- int beBackoffSlotLength = (int) hiveConf
- .getTimeVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH, TimeUnit.MILLISECONDS);
- TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket).processorFactory(processorFactory)
- .transportFactory(transportFactory).protocolFactory(new TBinaryProtocol.Factory())
- .inputProtocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize))
- .requestTimeout(requestTimeout).requestTimeoutUnit(TimeUnit.SECONDS).beBackoffSlotLength(beBackoffSlotLength)
- .beBackoffSlotLengthUnit(TimeUnit.MILLISECONDS).executorService(executorService);
-
- // TCP Server
- server = new TThreadPoolServer(sargs);
- server.setServerEventHandler(new TServerEventHandler() {
- @Override
- public ServerContext createContext(TProtocol input, TProtocol output) {
- return new ThriftCLIServerContext();
- }
-
- @Override
- public void deleteContext(ServerContext serverContext, TProtocol input, TProtocol output) {
- ThriftCLIServerContext context = (ThriftCLIServerContext) serverContext;
- SessionHandle sessionHandle = context.getSessionHandle();
- if (sessionHandle != null) {
- LOG.info("Session disconnected without closing properly. ");
- try {
- boolean close = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_CLOSE_SESSION_ON_DISCONNECT);
- LOG.info("Closing the session: " + sessionHandle);
- if (close) {
- cliService.closeSession(sessionHandle);
- }
- } catch (HiveSQLException e) {
- LOG.warn("Failed to close session: " + e, e);
- }
- }
- }
-
- @Override
- public void preServe() {
- }
-
- @Override
- public void processContext(ServerContext serverContext, TTransport input, TTransport output) {
- currentServerContext.set(serverContext);
- }
- });
- String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port " + portNum + " with "
- + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
- LOG.info(msg);
- } catch (Exception e) {
- throw new RuntimeException("Failed to init thrift server", e);
- }
- }
-
- @Override
- public void run() {
- try {
- server.serve();
- } catch (Throwable t) {
- if (t instanceof InterruptedException) {
- // This is likely a shutdown
- LOG.info("Caught " + t.getClass().getSimpleName() + ". Shutting down.");
- } else {
- LOG.error("Exception caught by " + this.getClass().getSimpleName() +
- ". Exiting.", t);
- }
- System.exit(-1);
- }
- }
-
- @Override
- protected void stopServer() {
- server.stop();
- server = null;
- LOG.info("Thrift server has stopped");
- }
-
-}
[3/5] incubator-livy git commit: [LIVY-502] Remove dependency on
hive-exec
Posted by va...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyCLIService.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyCLIService.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyCLIService.scala
index 5289354..725bdc8 100644
--- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyCLIService.scala
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyCLIService.scala
@@ -22,40 +22,36 @@ import java.util
import java.util.concurrent.{CancellationException, ExecutionException, TimeoutException, TimeUnit}
import javax.security.auth.login.LoginException
-import scala.collection.JavaConverters._
-
-import org.apache.hadoop.hive.common.log.ProgressMonitor
-import org.apache.hadoop.hive.conf.HiveConf
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hadoop.hive.ql.parse.ParseUtils
-import org.apache.hadoop.hive.shims.Utils
-import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hive.service.{CompositeService, ServiceException}
-import org.apache.hive.service.auth.HiveAuthFactory
+import org.apache.hadoop.security.{SecurityUtil, UserGroupInformation}
+import org.apache.hive.service.ServiceException
import org.apache.hive.service.cli._
-import org.apache.hive.service.cli.operation.Operation
import org.apache.hive.service.rpc.thrift.{TOperationHandle, TProtocolVersion}
-import org.apache.livy.{LIVY_VERSION, Logging}
+import org.apache.livy.{LIVY_VERSION, LivyConf, Logging}
+import org.apache.livy.thriftserver.auth.AuthFactory
+import org.apache.livy.thriftserver.operation.{Operation, OperationStatus}
+import org.apache.livy.thriftserver.serde.ThriftResultSet
+import org.apache.livy.thriftserver.types.Schema
class LivyCLIService(server: LivyThriftServer)
- extends CompositeService(classOf[LivyCLIService].getName) with ICLIService with Logging {
+ extends ThriftService(classOf[LivyCLIService].getName) with Logging {
import LivyCLIService._
private var sessionManager: LivyThriftSessionManager = _
private var defaultFetchRows: Int = _
private var serviceUGI: UserGroupInformation = _
private var httpUGI: UserGroupInformation = _
+ private var maxTimeout: Long = _
- override def init(hiveConf: HiveConf): Unit = {
- sessionManager = new LivyThriftSessionManager(server, hiveConf)
+ override def init(livyConf: LivyConf): Unit = {
+ sessionManager = new LivyThriftSessionManager(server, livyConf)
addService(sessionManager)
- defaultFetchRows =
- hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE)
+ defaultFetchRows = livyConf.getInt(LivyConf.THRIFT_RESULTSET_DEFAULT_FETCH_SIZE)
+ maxTimeout = livyConf.getTimeAsMs(LivyConf.THRIFT_LONG_POLLING_TIMEOUT)
// If the hadoop cluster is secure, do a kerberos login for the service from the keytab
if (UserGroupInformation.isSecurityEnabled) {
try {
- serviceUGI = Utils.getUGI
+ serviceUGI = UserGroupInformation.getCurrentUser
} catch {
case e: IOException =>
throw new ServiceException("Unable to login to kerberos with given principal/keytab", e)
@@ -63,19 +59,20 @@ class LivyCLIService(server: LivyThriftServer)
throw new ServiceException("Unable to login to kerberos with given principal/keytab", e)
}
// Also try creating a UGI object for the SPNego principal
- val principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_PRINCIPAL)
- val keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_KEYTAB)
+ val principal = livyConf.get(LivyConf.AUTH_KERBEROS_PRINCIPAL)
+ val keyTabFile = livyConf.get(LivyConf.AUTH_KERBEROS_KEYTAB)
if (principal.isEmpty || keyTabFile.isEmpty) {
info(s"SPNego httpUGI not created, SPNegoPrincipal: $principal, ketabFile: $keyTabFile")
} else try {
- httpUGI = HiveAuthFactory.loginFromSpnegoKeytabAndReturnUGI(hiveConf)
+ httpUGI = UserGroupInformation.loginUserFromKeytabAndReturnUGI(
+ SecurityUtil.getServerPrincipal(principal, "0.0.0.0"), keyTabFile)
info("SPNego httpUGI successfully created.")
} catch {
case e: IOException =>
warn("SPNego httpUGI creation failed: ", e)
}
}
- super.init(hiveConf)
+ super.init(livyConf)
}
def getServiceUGI: UserGroupInformation = this.serviceUGI
@@ -85,7 +82,7 @@ class LivyCLIService(server: LivyThriftServer)
def getSessionManager: LivyThriftSessionManager = sessionManager
@throws[HiveSQLException]
- override def getInfo(sessionHandle: SessionHandle, getInfoType: GetInfoType): GetInfoValue = {
+ def getInfo(sessionHandle: SessionHandle, getInfoType: GetInfoType): GetInfoValue = {
getInfoType match {
case GetInfoType.CLI_SERVER_NAME => new GetInfoValue("Livy JDBC")
case GetInfoType.CLI_DBMS_NAME => new GetInfoValue("Livy JDBC")
@@ -95,7 +92,7 @@ class LivyCLIService(server: LivyThriftServer)
case GetInfoType.CLI_MAX_SCHEMA_NAME_LEN => new GetInfoValue(128)
case GetInfoType.CLI_MAX_TABLE_NAME_LEN => new GetInfoValue(128)
case GetInfoType.CLI_ODBC_KEYWORDS =>
- new GetInfoValue(ParseUtils.getKeywords(LivyCLIService.ODBC_KEYWORDS))
+ new GetInfoValue(getKeywords)
case _ => throw new HiveSQLException(s"Unrecognized GetInfoType value: $getInfoType")
}
}
@@ -128,7 +125,7 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def openSession(
+ def openSession(
username: String,
password: String,
configuration: util.Map[String, String]): SessionHandle = {
@@ -139,7 +136,7 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def openSessionWithImpersonation(
+ def openSessionWithImpersonation(
username: String,
password: String,
configuration: util.Map[String, String], delegationToken: String): SessionHandle = {
@@ -150,13 +147,13 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def closeSession(sessionHandle: SessionHandle): Unit = {
+ def closeSession(sessionHandle: SessionHandle): Unit = {
sessionManager.closeSession(sessionHandle)
debug(sessionHandle + ": closeSession()")
}
@throws[HiveSQLException]
- override def executeStatement(
+ def executeStatement(
sessionHandle: SessionHandle,
statement: String,
confOverlay: util.Map[String, String]): OperationHandle = {
@@ -167,7 +164,7 @@ class LivyCLIService(server: LivyThriftServer)
* Execute statement on the server with a timeout. This is a blocking call.
*/
@throws[HiveSQLException]
- override def executeStatement(
+ def executeStatement(
sessionHandle: SessionHandle,
statement: String,
confOverlay: util.Map[String, String],
@@ -179,7 +176,7 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def executeStatementAsync(
+ def executeStatementAsync(
sessionHandle: SessionHandle,
statement: String,
confOverlay: util.Map[String, String]): OperationHandle = {
@@ -190,7 +187,7 @@ class LivyCLIService(server: LivyThriftServer)
* Execute statement asynchronously on the server with a timeout. This is a non-blocking call
*/
@throws[HiveSQLException]
- override def executeStatementAsync(
+ def executeStatementAsync(
sessionHandle: SessionHandle,
statement: String,
confOverlay: util.Map[String, String],
@@ -202,19 +199,19 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def getTypeInfo(sessionHandle: SessionHandle): OperationHandle = {
+ def getTypeInfo(sessionHandle: SessionHandle): OperationHandle = {
debug(sessionHandle + ": getTypeInfo()")
sessionManager.operationManager.getTypeInfo(sessionHandle)
}
@throws[HiveSQLException]
- override def getCatalogs(sessionHandle: SessionHandle): OperationHandle = {
+ def getCatalogs(sessionHandle: SessionHandle): OperationHandle = {
debug(sessionHandle + ": getCatalogs()")
sessionManager.operationManager.getCatalogs(sessionHandle)
}
@throws[HiveSQLException]
- override def getSchemas(
+ def getSchemas(
sessionHandle: SessionHandle,
catalogName: String,
schemaName: String): OperationHandle = {
@@ -223,7 +220,7 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def getTables(
+ def getTables(
sessionHandle: SessionHandle,
catalogName: String,
schemaName: String,
@@ -234,13 +231,13 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def getTableTypes(sessionHandle: SessionHandle): OperationHandle = {
+ def getTableTypes(sessionHandle: SessionHandle): OperationHandle = {
debug(sessionHandle + ": getTableTypes()")
sessionManager.operationManager.getTableTypes(sessionHandle)
}
@throws[HiveSQLException]
- override def getColumns(
+ def getColumns(
sessionHandle: SessionHandle,
catalogName: String,
schemaName: String,
@@ -251,7 +248,7 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def getFunctions(
+ def getFunctions(
sessionHandle: SessionHandle,
catalogName: String,
schemaName: String,
@@ -261,7 +258,7 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def getPrimaryKeys(
+ def getPrimaryKeys(
sessionHandle: SessionHandle,
catalog: String,
schema: String,
@@ -271,7 +268,7 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def getCrossReference(
+ def getCrossReference(
sessionHandle: SessionHandle,
primaryCatalog: String,
primarySchema: String,
@@ -284,7 +281,7 @@ class LivyCLIService(server: LivyThriftServer)
}
@throws[HiveSQLException]
- override def getOperationStatus(
+ def getOperationStatus(
opHandle: OperationHandle,
getProgressUpdate: Boolean): OperationStatus = {
val operation: Operation = sessionManager.operationManager.getOperation(opHandle)
@@ -294,10 +291,6 @@ class LivyCLIService(server: LivyThriftServer)
* However, if the background operation is complete, we return immediately.
*/
if (operation.shouldRunAsync) {
- val maxTimeout: Long = HiveConf.getTimeVar(
- getHiveConf,
- HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT,
- TimeUnit.MILLISECONDS)
val elapsed: Long = System.currentTimeMillis - operation.getBeginTime
// A step function to increase the polling timeout by 500 ms every 10 sec,
// starting from 500 ms up to HIVE_SERVER2_LONG_POLLING_TIMEOUT
@@ -321,76 +314,75 @@ class LivyCLIService(server: LivyThriftServer)
// In this case, the call might return sooner than long polling timeout
}
}
- val opStatus: OperationStatus = operation.getStatus
+ val opStatus = operation.getStatus
debug(opHandle + ": getOperationStatus()")
- opStatus.setJobProgressUpdate(new JobProgressUpdate(ProgressMonitor.NULL))
opStatus
}
@throws[HiveSQLException]
- override def cancelOperation(opHandle: OperationHandle): Unit = {
+ def cancelOperation(opHandle: OperationHandle): Unit = {
sessionManager.operationManager.cancelOperation(opHandle)
debug(opHandle + ": cancelOperation()")
}
@throws[HiveSQLException]
- override def closeOperation(opHandle: OperationHandle): Unit = {
+ def closeOperation(opHandle: OperationHandle): Unit = {
sessionManager.operationManager.closeOperation(opHandle)
debug(opHandle + ": closeOperation")
}
@throws[HiveSQLException]
- override def getResultSetMetadata(opHandle: OperationHandle): TableSchema = {
+ def getResultSetMetadata(opHandle: OperationHandle): Schema = {
debug(opHandle + ": getResultSetMetadata()")
sessionManager.operationManager.getOperation(opHandle).getResultSetSchema
}
@throws[HiveSQLException]
- override def fetchResults(opHandle: OperationHandle): RowSet = {
+ def fetchResults(opHandle: OperationHandle): ThriftResultSet = {
fetchResults(
opHandle, Operation.DEFAULT_FETCH_ORIENTATION, defaultFetchRows, FetchType.QUERY_OUTPUT)
}
@throws[HiveSQLException]
- override def fetchResults(
+ def fetchResults(
opHandle: OperationHandle,
orientation: FetchOrientation,
maxRows: Long,
- fetchType: FetchType): RowSet = {
+ fetchType: FetchType): ThriftResultSet = {
debug(opHandle + ": fetchResults()")
sessionManager.operationManager.fetchResults(opHandle, orientation, maxRows, fetchType)
}
@throws[HiveSQLException]
- override def getDelegationToken(
+ def getDelegationToken(
sessionHandle: SessionHandle,
- authFactory: HiveAuthFactory,
+ authFactory: AuthFactory,
owner: String,
renewer: String): String = {
throw new HiveSQLException("Operation not yet supported.")
}
@throws[HiveSQLException]
- override def setApplicationName(sh: SessionHandle, value: String): Unit = {
+ def setApplicationName(sh: SessionHandle, value: String): Unit = {
throw new HiveSQLException("Operation not yet supported.")
}
- override def cancelDelegationToken(
+ def cancelDelegationToken(
sessionHandle: SessionHandle,
- authFactory: HiveAuthFactory,
+ authFactory: AuthFactory,
tokenStr: String): Unit = {
throw new HiveSQLException("Operation not yet supported.")
}
- override def renewDelegationToken(
+ def renewDelegationToken(
sessionHandle: SessionHandle,
- authFactory: HiveAuthFactory,
+ authFactory: AuthFactory,
tokenStr: String): Unit = {
throw new HiveSQLException("Operation not yet supported.")
}
@throws[HiveSQLException]
- override def getQueryId(opHandle: TOperationHandle): String = {
+ def getQueryId(opHandle: TOperationHandle): String = {
throw new HiveSQLException("Operation not yet supported.")
}
}
@@ -428,5 +420,36 @@ object LivyCLIService {
"TIMESTAMP", "TIMEZONE_HOUR", "TIMEZONE_MINUTE", "TO", "TRAILING", "TRANSACTION", "TRANSLATE",
"TRANSLATION", "TRIM", "TRUE", "UNION", "UNIQUE", "UNKNOWN", "UPDATE", "UPPER", "USAGE",
"USER", "USING", "VALUE", "VALUES", "VARCHAR", "VARYING", "VIEW", "WHEN", "WHENEVER", "WHERE",
- "WITH", "WORK", "WRITE", "YEAR", "ZONE").asJava
+ "WITH", "WORK", "WRITE", "YEAR", "ZONE")
+
+ // scalastyle:off line.size.limit
+ // https://github.com/apache/spark/blob/515708d5f33d5acdb4206c626192d1838f8e691f/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4#L744-L1013
+ // scalastyle:on line.size.limit
+ private val SPARK_KEYWORDS = Seq("SELECT", "FROM", "ADD", "AS", "ALL", "ANY", "DISTINCT",
+ "WHERE", "GROUP", "BY", "GROUPING", "SETS", "CUBE", "ROLLUP", "ORDER", "HAVING", "LIMIT",
+ "AT", "OR", "AND", "IN", "NOT", "NO", "EXISTS", "BETWEEN", "LIKE", "RLIKE", "REGEXP", "IS",
+ "NULL", "TRUE", "FALSE", "NULLS", "ASC", "DESC", "FOR", "INTERVAL", "CASE", "WHEN", "THEN",
+ "ELSE", "END", "JOIN", "CROSS", "OUTER", "INNER", "LEFT", "SEMI", "RIGHT", "FULL", "NATURAL",
+ "ON", "PIVOT", "LATERAL", "WINDOW", "OVER", "PARTITION", "RANGE", "ROWS", "UNBOUNDED",
+ "PRECEDING", "FOLLOWING", "CURRENT", "FIRST", "AFTER", "LAST", "ROW", "WITH", "VALUES",
+ "CREATE", "TABLE", "DIRECTORY", "VIEW", "REPLACE", "INSERT", "DELETE", "INTO", "DESCRIBE",
+ "EXPLAIN", "FORMAT", "LOGICAL", "CODEGEN", "COST", "CAST", "SHOW", "TABLES", "COLUMNS",
+ "COLUMN", "USE", "PARTITIONS", "FUNCTIONS", "DROP", "UNION", "EXCEPT", "MINUS", "INTERSECT",
+ "TO", "TABLESAMPLE", "STRATIFY", "ALTER", "RENAME", "ARRAY", "MAP", "STRUCT", "COMMENT", "SET",
+ "RESET", "DATA", "START", "TRANSACTION", "COMMIT", "ROLLBACK", "MACRO", "IGNORE", "BOTH",
+ "LEADING", "TRAILING", "IF", "POSITION", "EXTRACT", "DIV", "PERCENT", "BUCKET", "OUT", "OF",
+ "SORT", "CLUSTER", "DISTRIBUTE", "OVERWRITE", "TRANSFORM", "REDUCE", "USING", "SERDE",
+ "SERDEPROPERTIES", "RECORDREADER", "RECORDWRITER", "DELIMITED", "FIELDS", "TERMINATED",
+ "COLLECTION", "ITEMS", "KEYS", "ESCAPED", "LINES", "SEPARATED", "FUNCTION", "EXTENDED",
+ "REFRESH", "CLEAR", "CACHE", "UNCACHE", "LAZY", "FORMATTED", "GLOBAL", "TEMPORARY", "TEMP",
+ "OPTIONS", "UNSET", "TBLPROPERTIES", "DBPROPERTIES", "BUCKETS", "SKEWED", "STORED",
+ "DIRECTORIES", "LOCATION", "EXCHANGE", "ARCHIVE", "UNARCHIVE", "FILEFORMAT", "TOUCH",
+ "COMPACT", "CONCATENATE", "CHANGE", "CASCADE", "RESTRICT", "CLUSTERED", "SORTED", "PURGE",
+ "INPUTFORMAT", "OUTPUTFORMAT", "DATABASE", "SCHEMA", "DATABASES", "SCHEMAS", "DFS", "TRUNCATE",
+ "ANALYZE", "COMPUTE", "LIST", "STATISTICS", "PARTITIONED", "EXTERNAL", "DEFINED", "REVOKE",
+ "GRANT", "LOCK", "UNLOCK", "MSCK", "REPAIR", "RECOVER", "EXPORT", "IMPORT", "LOAD", "ROLE",
+ "ROLES", "COMPACTIONS", "PRINCIPALS", "TRANSACTIONS", "INDEX", "INDEXES", "LOCKS", "OPTION",
+ "ANTI", "LOCAL", "INPATH")
+
+ def getKeywords: String = SPARK_KEYWORDS.filter(ODBC_KEYWORDS.contains).mkString(",")
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyExecuteStatementOperation.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyExecuteStatementOperation.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyExecuteStatementOperation.scala
index c2c4716..a067788 100644
--- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyExecuteStatementOperation.scala
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyExecuteStatementOperation.scala
@@ -18,31 +18,27 @@
package org.apache.livy.thriftserver
import java.security.PrivilegedExceptionAction
-import java.util
-import java.util.{Map => JMap}
import java.util.concurrent.{ConcurrentLinkedQueue, RejectedExecutionException}
import scala.collection.mutable
-import scala.collection.JavaConverters._
import scala.util.control.NonFatal
-import org.apache.hadoop.hive.serde2.thrift.{ColumnBuffer => ThriftColumnBuffer}
-import org.apache.hadoop.hive.shims.Utils
+import org.apache.hadoop.security.UserGroupInformation
import org.apache.hive.service.cli._
-import org.apache.hive.service.cli.operation.Operation
import org.apache.livy.Logging
import org.apache.livy.thriftserver.SessionStates._
+import org.apache.livy.thriftserver.operation.Operation
import org.apache.livy.thriftserver.rpc.RpcClient
-import org.apache.livy.thriftserver.types.DataTypeUtils._
+import org.apache.livy.thriftserver.serde.ThriftResultSet
+import org.apache.livy.thriftserver.types.{BasicDataType, DataTypeUtils, Field, Schema}
class LivyExecuteStatementOperation(
sessionHandle: SessionHandle,
statement: String,
- confOverlay: JMap[String, String],
runInBackground: Boolean = true,
sessionManager: LivyThriftSessionManager)
- extends Operation(sessionHandle, confOverlay, OperationType.EXECUTE_STATEMENT)
+ extends Operation(sessionHandle, OperationType.EXECUTE_STATEMENT)
with Logging {
/**
@@ -62,31 +58,23 @@ class LivyExecuteStatementOperation(
}
private var rowOffset = 0L
- private def statementId: String = getHandle.getHandleIdentifier.toString
+ private def statementId: String = opHandle.getHandleIdentifier.toString
private def rpcClientValid: Boolean =
sessionManager.livySessionState(sessionHandle) == CREATION_SUCCESS && rpcClient.isValid
- override def getNextRowSet(order: FetchOrientation, maxRowsL: Long): RowSet = {
- validateDefaultFetchOrientation(order)
- assertState(util.Arrays.asList(OperationState.FINISHED))
+ override def getNextRowSet(order: FetchOrientation, maxRowsL: Long): ThriftResultSet = {
+ validateFetchOrientation(order)
+ assertState(Seq(OperationState.FINISHED))
setHasResultSet(true)
// maxRowsL here typically maps to java.sql.Statement.getFetchSize, which is an int
val maxRows = maxRowsL.toInt
val resultSet = rpcClient.fetchResult(sessionHandle, statementId, maxRows).get()
-
- val thriftColumns = resultSet.getColumns().map { col =>
- new ThriftColumnBuffer(toHiveThriftType(col.getType()), col.getNulls(), col.getValues())
- }
- val result = new ColumnBasedSet(
- toHiveTableSchema(resultSet.getSchema()).toTypeDescriptors,
- thriftColumns.toList.asJava,
- rowOffset)
- if (resultSet.getColumns() != null && resultSet.getColumns().length > 0) {
- rowOffset += resultSet.getColumns()(0).size()
- }
- result
+ val livyColumnResultSet = ThriftResultSet(resultSet)
+ livyColumnResultSet.setRowOffset(rowOffset)
+ rowOffset += livyColumnResultSet.numRows
+ livyColumnResultSet
}
override def runInternal(): Unit = {
@@ -96,7 +84,7 @@ class LivyExecuteStatementOperation(
if (!runInBackground) {
execute()
} else {
- val livyServiceUGI = Utils.getUGI
+ val livyServiceUGI = UserGroupInformation.getCurrentUser
// Runnable impl to call runInternal asynchronously,
// from a different thread
@@ -153,7 +141,7 @@ class LivyExecuteStatementOperation(
rpcClient.executeSql(sessionHandle, statementId, statement).get()
} catch {
case e: Throwable =>
- val currentState = getStatus.getState
+ val currentState = getStatus.state
info(s"Error executing query, currentState $currentState, ", e)
setState(OperationState.ERROR)
throw new HiveSQLException(e)
@@ -171,14 +159,17 @@ class LivyExecuteStatementOperation(
cleanup(state)
}
- def getResultSetSchema: TableSchema = {
- val tableSchema = toHiveTableSchema(
+ override def shouldRunAsync: Boolean = runInBackground
+
+ override def getResultSetSchema: Schema = {
+ val tableSchema = DataTypeUtils.schemaFromSparkJson(
rpcClient.fetchResultSchema(sessionHandle, statementId).get())
// Workaround for operations returning an empty schema (eg. CREATE, INSERT, ...)
- if (tableSchema.getSize == 0) {
- tableSchema.addStringColumn("Result", "")
+ if (!tableSchema.fields.isEmpty) {
+ tableSchema
+ } else {
+ Schema(Field("Result", BasicDataType("string"), ""))
}
- tableSchema
}
private def cleanup(state: OperationState) {
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyOperationManager.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyOperationManager.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyOperationManager.scala
index c71171a..e6d48ff 100644
--- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyOperationManager.scala
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyOperationManager.scala
@@ -23,12 +23,12 @@ import java.util.concurrent.ConcurrentHashMap
import scala.collection.mutable
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hadoop.hive.metastore.api.{FieldSchema, Schema}
import org.apache.hive.service.cli._
-import org.apache.hive.service.cli.operation.{GetCatalogsOperation, GetTableTypesOperation, GetTypeInfoOperation, Operation}
-import org.apache.livy.Logging
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.thriftserver.operation._
+import org.apache.livy.thriftserver.serde.ThriftResultSet
+import org.apache.livy.thriftserver.session.DataType
class LivyOperationManager(val livyThriftSessionManager: LivyThriftSessionManager)
extends Logging {
@@ -37,12 +37,15 @@ class LivyOperationManager(val livyThriftSessionManager: LivyThriftSessionManage
private val sessionToOperationHandles =
new mutable.HashMap[SessionHandle, mutable.Set[OperationHandle]]()
+ private val operationTimeout =
+ livyThriftSessionManager.livyConf.getTimeAsMs(LivyConf.THRIFT_IDLE_OPERATION_TIMEOUT)
+
private def addOperation(operation: Operation, sessionHandle: SessionHandle): Unit = {
- handleToOperation.put(operation.getHandle, operation)
+ handleToOperation.put(operation.opHandle, operation)
sessionToOperationHandles.synchronized {
val set = sessionToOperationHandles.getOrElseUpdate(sessionHandle,
new mutable.HashSet[OperationHandle])
- set += operation.getHandle
+ set += operation.opHandle
}
}
@@ -52,7 +55,7 @@ class LivyOperationManager(val livyThriftSessionManager: LivyThriftSessionManage
if (operation == null) {
throw new HiveSQLException(s"Operation does not exist: $operationHandle")
}
- val sessionHandle = operation.getSessionHandle
+ val sessionHandle = operation.sessionHandle
sessionToOperationHandles.synchronized {
sessionToOperationHandles(sessionHandle) -= operationHandle
if (sessionToOperationHandles(sessionHandle).isEmpty) {
@@ -74,7 +77,7 @@ class LivyOperationManager(val livyThriftSessionManager: LivyThriftSessionManage
opHandles.flatMap { handle =>
// Some operations may have finished and been removed since we got them.
Option(handleToOperation.get(handle))
- }.filter(_.isTimedOut(currentTime))
+ }.filter(_.isTimedOut(currentTime, operationTimeout))
}
@throws[HiveSQLException]
@@ -95,7 +98,6 @@ class LivyOperationManager(val livyThriftSessionManager: LivyThriftSessionManage
val op = new LivyExecuteStatementOperation(
sessionHandle,
statement,
- confOverlay,
runAsync,
livyThriftSessionManager)
addOperation(op, sessionHandle)
@@ -107,16 +109,13 @@ class LivyOperationManager(val livyThriftSessionManager: LivyThriftSessionManage
def getOperationLogRowSet(
opHandle: OperationHandle,
orientation: FetchOrientation,
- maxRows: Long): RowSet = {
- val tableSchema = new TableSchema(LivyOperationManager.LOG_SCHEMA)
- val session = livyThriftSessionManager.getSessionInfo(getOperation(opHandle).getSessionHandle)
- val logs = RowSetFactory.create(tableSchema, session.protocolVersion, false)
-
- if (!livyThriftSessionManager.getHiveConf.getBoolVar(
- ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED)) {
- warn("Try to get operation log when " +
- ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED.varname +
- " is false, no log will be returned. ")
+ maxRows: Long): ThriftResultSet = {
+ val session = livyThriftSessionManager.getSessionInfo(getOperation(opHandle).sessionHandle)
+ val logs = ThriftResultSet(LivyOperationManager.LOG_SCHEMA, session.protocolVersion)
+
+ if (!livyThriftSessionManager.livyConf.getBoolean(LivyConf.THRIFT_LOG_OPERATION_ENABLED)) {
+ warn(s"Try to get operation log when ${LivyConf.THRIFT_LOG_OPERATION_ENABLED.key} is " +
+ "false, no log will be returned.")
} else {
// Get the operation log. This is implemented only for LivyExecuteStatementOperation
val operation = getOperation(opHandle)
@@ -149,7 +148,7 @@ class LivyOperationManager(val livyThriftSessionManager: LivyThriftSessionManage
var opHandle: OperationHandle = null
try {
val operation = operationCreator
- opHandle = operation.getHandle
+ opHandle = operation.opHandle
operation.run()
opHandle
} catch {
@@ -194,9 +193,9 @@ class LivyOperationManager(val livyThriftSessionManager: LivyThriftSessionManage
@throws[HiveSQLException]
def cancelOperation(opHandle: OperationHandle, errMsg: String): Unit = {
val operation = getOperation(opHandle)
- val opState = operation.getStatus.getState
+ val opState = operation.getStatus.state
if (opState.isTerminal) {
- // Cancel should be a no-op
+ // Cancel should be a no-op either case
debug(s"$opHandle: Operation is already aborted in state - $opState")
} else {
debug(s"$opHandle: Attempting to cancel from state - $opState")
@@ -223,7 +222,7 @@ class LivyOperationManager(val livyThriftSessionManager: LivyThriftSessionManage
opHandle: OperationHandle,
orientation: FetchOrientation,
maxRows: Long,
- fetchType: FetchType): RowSet = {
+ fetchType: FetchType): ThriftResultSet = {
if (fetchType == FetchType.QUERY_OUTPUT) {
getOperation(opHandle).getNextRowSet(orientation, maxRows)
} else {
@@ -233,12 +232,5 @@ class LivyOperationManager(val livyThriftSessionManager: LivyThriftSessionManage
}
object LivyOperationManager {
- val LOG_SCHEMA: Schema = {
- val schema = new Schema
- val fieldSchema = new FieldSchema
- fieldSchema.setName("operation_log")
- fieldSchema.setType("string")
- schema.addToFieldSchemas(fieldSchema)
- schema
- }
+ val LOG_SCHEMA: Array[DataType] = Array(DataType.STRING)
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyThriftServer.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyThriftServer.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyThriftServer.scala
index c670217..daf7b82 100644
--- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyThriftServer.scala
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyThriftServer.scala
@@ -19,17 +19,14 @@ package org.apache.livy.thriftserver
import java.security.PrivilegedExceptionAction
-import scala.collection.JavaConverters._
-
-import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hive.service.server.HiveServer2
import org.apache.livy.{LivyConf, Logging}
import org.apache.livy.server.AccessManager
import org.apache.livy.server.interactive.InteractiveSession
import org.apache.livy.server.recovery.SessionStore
import org.apache.livy.sessions.InteractiveSessionManager
+import org.apache.livy.thriftserver.cli.{ThriftBinaryCLIService, ThriftHttpCLIService}
/**
* The main entry point for the Livy thrift server leveraging HiveServer2. Starts up a
@@ -41,21 +38,6 @@ object LivyThriftServer extends Logging {
private[thriftserver] var thriftServerThread: Thread = _
private var thriftServer: LivyThriftServer = _
- private def hiveConf(livyConf: LivyConf): HiveConf = {
- val conf = new HiveConf()
- // Remove all configs coming from hive-site.xml which may be in the classpath for the Spark
- // applications to run.
- conf.getAllProperties.asScala.filter(_._1.startsWith("hive.")).foreach { case (key, _) =>
- conf.unset(key)
- }
- livyConf.asScala.foreach {
- case nameAndValue if nameAndValue.getKey.startsWith("livy.hive") =>
- conf.set(nameAndValue.getKey.stripPrefix("livy."), nameAndValue.getValue)
- case _ => // Ignore
- }
- conf
- }
-
def start(
livyConf: LivyConf,
livySessionManager: InteractiveSessionManager,
@@ -97,7 +79,7 @@ object LivyThriftServer extends Logging {
}
private def doStart(livyConf: LivyConf): Unit = {
- thriftServer.init(hiveConf(livyConf))
+ thriftServer.init(livyConf)
thriftServer.start()
}
@@ -114,6 +96,11 @@ object LivyThriftServer extends Logging {
thriftServer.stop()
thriftServer = null
}
+
+ def isHTTPTransportMode(livyConf: LivyConf): Boolean = {
+ val transportMode = livyConf.get(LivyConf.THRIFT_TRANSPORT_MODE)
+ transportMode != null && transportMode.equalsIgnoreCase("http")
+ }
}
@@ -121,17 +108,38 @@ class LivyThriftServer(
private[thriftserver] val livyConf: LivyConf,
private[thriftserver] val livySessionManager: InteractiveSessionManager,
private[thriftserver] val sessionStore: SessionStore,
- private[thriftserver] val accessManager: AccessManager) extends HiveServer2 {
- override def init(hiveConf: HiveConf): Unit = {
- this.cliService = new LivyCLIService(this)
- super.init(hiveConf)
+ private[thriftserver] val accessManager: AccessManager)
+ extends ThriftService(classOf[LivyThriftServer].getName) with Logging {
+
+ val cliService = new LivyCLIService(this)
+
+ override def init(livyConf: LivyConf): Unit = {
+ addService(cliService)
+ val server = this
+ val oomHook = new Runnable() {
+ override def run(): Unit = {
+ server.stop()
+ }
+ }
+ val thriftCLIService = if (LivyThriftServer.isHTTPTransportMode(livyConf)) {
+ new ThriftHttpCLIService(cliService, oomHook)
+ } else {
+ new ThriftBinaryCLIService(cliService, oomHook)
+ }
+ addService(thriftCLIService)
+ super.init(livyConf)
}
- private[thriftserver] def getSessionManager(): LivyThriftSessionManager = {
- this.cliService.asInstanceOf[LivyCLIService].getSessionManager
+ private[thriftserver] def getSessionManager = {
+ cliService.getSessionManager
}
def isAllowedToUse(user: String, session: InteractiveSession): Boolean = {
session.owner == user || accessManager.checkModifyPermissions(user)
}
+
+ override def stop(): Unit = {
+ info("Shutting down LivyThriftServer")
+ super.stop()
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyThriftSessionManager.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyThriftSessionManager.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyThriftSessionManager.scala
index 344a990..5be5536 100644
--- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyThriftSessionManager.scala
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/LivyThriftSessionManager.scala
@@ -31,10 +31,7 @@ import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.util.{Failure, Success, Try}
-import org.apache.hadoop.hive.conf.HiveConf
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hadoop.hive.shims.Utils
-import org.apache.hive.service.CompositeService
+import org.apache.hadoop.security.UserGroupInformation
import org.apache.hive.service.cli.{HiveSQLException, SessionHandle}
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup
@@ -47,8 +44,9 @@ import org.apache.livy.thriftserver.SessionStates._
import org.apache.livy.thriftserver.rpc.RpcClient
import org.apache.livy.utils.LivySparkUtils
-class LivyThriftSessionManager(val server: LivyThriftServer, hiveConf: HiveConf)
- extends CompositeService(classOf[LivyThriftSessionManager].getName) with Logging {
+
+class LivyThriftSessionManager(val server: LivyThriftServer, val livyConf: LivyConf)
+ extends ThriftService(classOf[LivyThriftSessionManager].getName) with Logging {
private[thriftserver] val operationManager = new LivyOperationManager(this)
private val sessionHandleToLivySession =
@@ -79,17 +77,13 @@ class LivyThriftSessionManager(val server: LivyThriftServer, hiveConf: HiveConf)
}
// Configs from Hive
- private val userLimit = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_LIMIT_CONNECTIONS_PER_USER)
- private val ipAddressLimit =
- hiveConf.getIntVar(ConfVars.HIVE_SERVER2_LIMIT_CONNECTIONS_PER_IPADDRESS)
+ private val userLimit = livyConf.getInt(LivyConf.THRIFT_LIMIT_CONNECTIONS_PER_USER)
+ private val ipAddressLimit = livyConf.getInt(LivyConf.THRIFT_LIMIT_CONNECTIONS_PER_IPADDRESS)
private val userIpAddressLimit =
- hiveConf.getIntVar(ConfVars.HIVE_SERVER2_LIMIT_CONNECTIONS_PER_USER_IPADDRESS)
- private val checkInterval = HiveConf.getTimeVar(
- hiveConf, ConfVars.HIVE_SERVER2_SESSION_CHECK_INTERVAL, TimeUnit.MILLISECONDS)
- private val sessionTimeout = HiveConf.getTimeVar(
- hiveConf, ConfVars.HIVE_SERVER2_IDLE_SESSION_TIMEOUT, TimeUnit.MILLISECONDS)
- private val checkOperation = HiveConf.getBoolVar(
- hiveConf, ConfVars.HIVE_SERVER2_IDLE_SESSION_CHECK_OPERATION)
+ livyConf.getInt(LivyConf.THRIFT_LIMIT_CONNECTIONS_PER_USER_IPADDRESS)
+ private val checkInterval = livyConf.getTimeAsMs(LivyConf.THRIFT_SESSION_CHECK_INTERVAL)
+ private val sessionTimeout = livyConf.getTimeAsMs(LivyConf.THRIFT_IDLE_SESSION_TIMEOUT)
+ private val checkOperation = livyConf.getBoolean(LivyConf.THRIFT_IDLE_SESSION_CHECK_OPERATION)
private var backgroundOperationPool: ThreadPoolExecutor = _
@@ -243,7 +237,7 @@ class LivyThriftSessionManager(val server: LivyThriftServer, hiveConf: HiveConf)
newSession
}
val futureLivySession = Future {
- val livyServiceUGI = Utils.getUGI
+ val livyServiceUGI = UserGroupInformation.getCurrentUser
var livySession: InteractiveSession = null
try {
livyServiceUGI.doAs(new PrivilegedExceptionAction[InteractiveSession] {
@@ -294,21 +288,20 @@ class LivyThriftSessionManager(val server: LivyThriftServer, hiveConf: HiveConf)
}
// Taken from Hive
- override def init(hiveConf: HiveConf): Unit = {
- createBackgroundOperationPool(hiveConf)
+ override def init(livyConf: LivyConf): Unit = {
+ createBackgroundOperationPool(livyConf)
info("Connections limit are user: {} ipaddress: {} user-ipaddress: {}",
userLimit, ipAddressLimit, userIpAddressLimit)
- super.init(hiveConf)
+ super.init(livyConf)
}
// Taken from Hive
- private def createBackgroundOperationPool(hiveConf: HiveConf): Unit = {
- val poolSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS)
+ private def createBackgroundOperationPool(livyConf: LivyConf): Unit = {
+ val poolSize = livyConf.getInt(LivyConf.THRIFT_ASYNC_EXEC_THREADS)
info("HiveServer2: Background operation thread pool size: " + poolSize)
- val poolQueueSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE)
+ val poolQueueSize = livyConf.getInt(LivyConf.THRIFT_ASYNC_EXEC_WAIT_QUEUE_SIZE)
info("HiveServer2: Background operation thread wait queue size: " + poolQueueSize)
- val keepAliveTime = HiveConf.getTimeVar(
- hiveConf, ConfVars.HIVE_SERVER2_ASYNC_EXEC_KEEPALIVE_TIME, TimeUnit.SECONDS)
+ val keepAliveTime = livyConf.getTimeAsMs(LivyConf.THRIFT_ASYNC_EXEC_KEEPALIVE_TIME) / 1000
info(s"HiveServer2: Background operation thread keepalive time: $keepAliveTime seconds")
// Create a thread pool with #poolSize threads
// Threads terminate when they are idle for more than the keepAliveTime
@@ -364,11 +357,11 @@ class LivyThriftSessionManager(val server: LivyThriftServer, hiveConf: HiveConf)
if (operations.nonEmpty) {
operations.foreach { op =>
try {
- warn(s"Operation ${op.getHandle} is timed-out and will be closed")
- operationManager.closeOperation(op.getHandle)
+ warn(s"Operation ${op.opHandle} is timed-out and will be closed")
+ operationManager.closeOperation(op.opHandle)
} catch {
case e: Exception =>
- warn("Exception is thrown closing timed-out operation: " + op.getHandle, e)
+ warn("Exception is thrown closing timed-out operation: " + op.opHandle, e)
}
}
}
@@ -405,14 +398,13 @@ class LivyThriftSessionManager(val server: LivyThriftServer, hiveConf: HiveConf)
shutdownTimeoutChecker()
if (backgroundOperationPool != null) {
backgroundOperationPool.shutdown()
- val timeout =
- hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT, TimeUnit.SECONDS)
+ val timeout = livyConf.getTimeAsMs(LivyConf.THRIFT_ASYNC_EXEC_SHUTDOWN_TIMEOUT) / 1000
try {
backgroundOperationPool.awaitTermination(timeout, TimeUnit.SECONDS)
} catch {
case e: InterruptedException =>
- warn("HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT = " + timeout +
- " seconds has been exceeded. RUNNING background operations will be shut down", e)
+ warn(s"THRIFT_ASYNC_EXEC_SHUTDOWN_TIMEOUT = $timeout seconds has been exceeded. " +
+ "RUNNING background operations will be shut down", e)
}
backgroundOperationPool = null
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala
index 7669580..16f903a 100644
--- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala
@@ -38,6 +38,11 @@ class ThriftServerFactoryImpl extends ThriftServerFactory {
LivyThriftServer.start(livyConf, livySessionManager, sessionStore, accessManager)
}
+ override def stop(): Unit = {
+ assert(LivyThriftServer.getInstance.isDefined)
+ LivyThriftServer.getInstance.foreach(_.stop())
+ }
+
override def getServlet(basePath: String): Servlet = new ThriftJsonServlet(basePath)
override def getServletMappings: Seq[String] = Seq("/thriftserver/*")
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftService.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftService.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftService.scala
new file mode 100644
index 0000000..467592a
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftService.scala
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver
+
+import scala.collection.mutable
+
+import org.apache.hive.service.ServiceException
+
+import org.apache.livy.{LivyConf, Logging}
+
+/**
+ * Service states
+ */
+object STATE extends Enumeration {
+ type STATE = Value
+ val
+
+ /** Constructed but not initialized */
+ NOTINITED,
+
+ /** Initialized but not started or stopped */
+ INITED,
+
+ /** started and not stopped */
+ STARTED,
+
+ /** stopped. No further state transitions are permitted */
+ STOPPED = Value
+}
+
+class ThriftService(val name: String) extends Logging {
+ private val serviceList = new mutable.ListBuffer[ThriftService]
+
+ /**
+ * Service state: initially {@link STATE#NOTINITED}.
+ */
+ private var state = STATE.NOTINITED
+
+ /**
+ * Service start time. Will be zero until the service is started.
+ */
+ private var startTime = 0L
+
+ def getServices: Seq[ThriftService] = serviceList.toList
+
+ protected def addService(service: ThriftService): Unit = {
+ serviceList += service
+ }
+
+ protected def removeService(service: ThriftService): Unit = serviceList -= service
+
+ def init(conf: LivyConf): Unit = {
+ serviceList.foreach(_.init(conf))
+ ensureCurrentState(STATE.NOTINITED)
+ changeState(STATE.INITED)
+ info(s"Service:$getName is inited.")
+ }
+
+ def start(): Unit = {
+ var i = 0
+ try {
+ val n = serviceList.size
+ while (i < n) {
+ val service = serviceList(i)
+ service.start()
+ i += 1
+ }
+ startTime = System.currentTimeMillis
+ ensureCurrentState(STATE.INITED)
+ changeState(STATE.STARTED)
+ info(s"Service:$getName is started.")
+ } catch {
+ case e: Throwable =>
+ error("Error starting services " + getName, e)
+ // Note that the state of the failed service is still INITED and not
+ // STARTED. Even though the last service is not started completely, still
+ // call stop() on all services including failed service to make sure cleanup
+ // happens.
+ stop(i)
+ throw new ServiceException("Failed to Start " + getName, e)
+ }
+ }
+
+ def stop(): Unit = {
+ if (this.getServiceState == STATE.STOPPED) {
+ // The base composite-service is already stopped, don't do anything again.
+ return
+ }
+ if (serviceList.nonEmpty) stop(serviceList.size - 1)
+ if ((state == STATE.STOPPED) || (state == STATE.INITED) || (state == STATE.NOTINITED)) {
+ // already stopped, or else it was never
+ // started (eg another service failing canceled startup)
+ return
+ }
+ ensureCurrentState(STATE.STARTED)
+ changeState(STATE.STOPPED)
+ info(s"Service:$getName is stopped.")
+ }
+
+ private def stop(numOfServicesStarted: Int): Unit = {
+ // stop in reverse order of start
+ var i = numOfServicesStarted
+ while (i >= 0) {
+ val service = serviceList(i)
+ try {
+ service.stop()
+ } catch {
+ case t: Throwable => info("Error stopping " + service.getName, t)
+ }
+ i -= 1
+ }
+ }
+
+ def getServiceState: STATE.Value = state
+
+ def getName: String = name
+
+ def getStartTime: Long = startTime
+
+ /**
+ * Verify that that a service is in a given state.
+ *
+ * @param currentState
+ * the desired state
+ * @throws IllegalStateException
+ * if the service state is different from
+ * the desired state
+ */
+ private def ensureCurrentState(currentState: STATE.Value): Unit = {
+ if (state != currentState) {
+ throw new IllegalStateException(
+ s"For this operation, the current service state must be $currentState instead of $state")
+ }
+ }
+
+ /**
+ * Change to a new state.
+ *
+ * @param newState new service state
+ */
+ private def changeState(newState: STATE.Value): Unit = {
+ state = newState
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthBridgeServer.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthBridgeServer.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthBridgeServer.scala
new file mode 100644
index 0000000..d2091e6
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthBridgeServer.scala
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.auth
+
+import java.io.IOException
+import java.net.InetAddress
+import java.security.{PrivilegedAction, PrivilegedExceptionAction}
+import java.util
+import javax.security.auth.callback.{Callback, CallbackHandler, NameCallback, PasswordCallback, UnsupportedCallbackException}
+import javax.security.sasl.{AuthorizeCallback, RealmCallback, SaslServer}
+
+import org.apache.commons.codec.binary.Base64
+import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.security.{SaslRpcServer, UserGroupInformation}
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod
+import org.apache.hadoop.security.token.SecretManager.InvalidToken
+import org.apache.thrift.{TException, TProcessor}
+import org.apache.thrift.protocol.TProtocol
+import org.apache.thrift.transport.{TSaslServerTransport, TSocket, TTransport, TTransportException, TTransportFactory}
+
+import org.apache.livy.Logging
+
+/**
+ * The class is taken from Hive's `HadoopThriftAuthBridge.Server`. It bridges Thrift's SASL
+ * transports to Hadoop's SASL callback handlers and authentication classes.
+ */
+class AuthBridgeServer(private val secretManager: LivyDelegationTokenSecretManager) {
+ private val ugi = try {
+ UserGroupInformation.getCurrentUser
+ } catch {
+ case ioe: IOException => throw new TTransportException(ioe)
+ }
+
+ /**
+ * Create a TTransportFactory that, upon connection of a client socket,
+ * negotiates a Kerberized SASL transport. The resulting TTransportFactory
+ * can be passed as both the input and output transport factory when
+ * instantiating a TThreadPoolServer, for example.
+ *
+ * @param saslProps Map of SASL properties
+ */
+ @throws[TTransportException]
+ def createTransportFactory(saslProps: util.Map[String, String]): TTransportFactory = {
+ val transFactory: TSaslServerTransport.Factory = createSaslServerTransportFactory(saslProps)
+ new TUGIAssumingTransportFactory(transFactory, ugi)
+ }
+
+ /**
+ * Create a TSaslServerTransport.Factory that, upon connection of a client
+ * socket, negotiates a Kerberized SASL transport.
+ *
+ * @param saslProps Map of SASL properties
+ */
+ @throws[TTransportException]
+ def createSaslServerTransportFactory(
+ saslProps: util.Map[String, String]): TSaslServerTransport.Factory = {
+ // Parse out the kerberos principal, host, realm.
+ val kerberosName: String = ugi.getUserName
+ val names: Array[String] = SaslRpcServer.splitKerberosName(kerberosName)
+ if (names.length != 3) {
+ throw new TTransportException(s"Kerberos principal should have 3 parts: $kerberosName")
+ }
+ val transFactory: TSaslServerTransport.Factory = new TSaslServerTransport.Factory
+ transFactory.addServerDefinition(AuthMethod.KERBEROS.getMechanismName,
+ names(0), names(1), // two parts of kerberos principal
+ saslProps,
+ new SaslRpcServer.SaslGssCallbackHandler)
+ transFactory.addServerDefinition(AuthMethod.TOKEN.getMechanismName,
+ null,
+ SaslRpcServer.SASL_DEFAULT_REALM,
+ saslProps,
+ new SaslDigestCallbackHandler(secretManager))
+ transFactory
+ }
+
+ /**
+ * Wrap a TTransportFactory in such a way that, before processing any RPC, it
+ * assumes the UserGroupInformation of the user authenticated by
+ * the SASL transport.
+ */
+ def wrapTransportFactory(transFactory: TTransportFactory): TTransportFactory = {
+ new TUGIAssumingTransportFactory(transFactory, ugi)
+ }
+
+ /**
+ * Wrap a TProcessor in such a way that, before processing any RPC, it
+ * assumes the UserGroupInformation of the user authenticated by
+ * the SASL transport.
+ */
+ def wrapProcessor(processor: TProcessor): TProcessor = {
+ new TUGIAssumingProcessor(processor, secretManager, true)
+ }
+
+ /**
+ * Wrap a TProcessor to capture the client information like connecting userid, ip etc
+ */
+ def wrapNonAssumingProcessor(processor: TProcessor): TProcessor = {
+ new TUGIAssumingProcessor(processor, secretManager, false)
+ }
+
+ def getRemoteAddress: InetAddress = AuthBridgeServer.remoteAddress.get
+
+ def getRemoteUser: String = AuthBridgeServer.remoteUser.get
+
+ def getUserAuthMechanism: String = AuthBridgeServer.userAuthMechanism.get
+
+}
+
+/**
+ * A TransportFactory that wraps another one, but assumes a specified UGI
+ * before calling through.
+ *
+ * This is used on the server side to assume the server's Principal when accepting
+ * clients.
+ *
+ * This class is derived from Hive's one.
+ */
+private[auth] class TUGIAssumingTransportFactory(
+ val wrapped: TTransportFactory,
+ val ugi: UserGroupInformation) extends TTransportFactory {
+ assert(wrapped != null)
+ assert(ugi != null)
+
+ override def getTransport(trans: TTransport): TTransport = {
+ ugi.doAs(new PrivilegedAction[TTransport]() {
+ override def run: TTransport = wrapped.getTransport(trans)
+ })
+ }
+}
+
+/**
+ * CallbackHandler for SASL DIGEST-MD5 mechanism.
+ *
+ * This code is pretty much completely based on Hadoop's SaslRpcServer.SaslDigestCallbackHandler -
+ * the only reason we could not use that Hadoop class as-is was because it needs a
+ * Server.Connection.
+ */
+sealed class SaslDigestCallbackHandler(
+ val secretManager: LivyDelegationTokenSecretManager) extends CallbackHandler with Logging {
+ @throws[InvalidToken]
+ private def getPassword(tokenId: LivyDelegationTokenIdentifier): Array[Char] = {
+ encodePassword(secretManager.retrievePassword(tokenId))
+ }
+
+ private def encodePassword(password: Array[Byte]): Array[Char] = {
+ new String(Base64.encodeBase64(password)).toCharArray
+ }
+
+ @throws[InvalidToken]
+ @throws[UnsupportedCallbackException]
+ override def handle(callbacks: Array[Callback]): Unit = {
+ var nc: NameCallback = null
+ var pc: PasswordCallback = null
+ callbacks.foreach {
+ case ac: AuthorizeCallback =>
+ val authid: String = ac.getAuthenticationID
+ val authzid: String = ac.getAuthorizationID
+ if (authid == authzid) {
+ ac.setAuthorized(true)
+ } else {
+ ac.setAuthorized(false)
+ }
+ if (ac.isAuthorized) {
+ if (logger.isDebugEnabled) {
+ val username = SaslRpcServer.getIdentifier(authzid, secretManager).getUser.getUserName
+ debug(s"SASL server DIGEST-MD5 callback: setting canonicalized client ID: $username")
+ }
+ ac.setAuthorizedID(authzid)
+ }
+ case c: NameCallback => nc = c
+ case c: PasswordCallback => pc = c
+ case _: RealmCallback => // Do nothing.
+ case other =>
+ throw new UnsupportedCallbackException(other, "Unrecognized SASL DIGEST-MD5 Callback")
+ }
+ if (pc != null) {
+ val tokenIdentifier = SaslRpcServer.getIdentifier(nc.getDefaultName, secretManager)
+ val password: Array[Char] = getPassword(tokenIdentifier)
+ if (logger.isDebugEnabled) {
+ debug("SASL server DIGEST-MD5 callback: setting password for client: " +
+ tokenIdentifier.getUser)
+ }
+ pc.setPassword(password)
+ }
+ }
+}
+
+/**
+ * Processor that pulls the SaslServer object out of the transport, and assumes the remote user's
+ * UGI before calling through to the original processor.
+ *
+ * This is used on the server side to set the UGI for each specific call.
+ *
+ * This class is derived from Hive's one.
+ */
+sealed class TUGIAssumingProcessor(
+ val wrapped: TProcessor,
+ val secretManager: LivyDelegationTokenSecretManager,
+ var useProxy: Boolean) extends TProcessor with Logging {
+
+ @throws[TException]
+ override def process(inProt: TProtocol, outProt: TProtocol): Boolean = {
+ val trans = inProt.getTransport
+ if (!trans.isInstanceOf[TSaslServerTransport]) {
+ throw new TException(s"Unexpected non-SASL transport ${trans.getClass}")
+ }
+ val saslTrans: TSaslServerTransport = trans.asInstanceOf[TSaslServerTransport]
+ val saslServer: SaslServer = saslTrans.getSaslServer
+ val authId: String = saslServer.getAuthorizationID
+ debug(s"AUTH ID ======> $authId")
+ var endUser = authId
+ val socket = saslTrans.getUnderlyingTransport.asInstanceOf[TSocket].getSocket
+ AuthBridgeServer.remoteAddress.set(socket.getInetAddress)
+ val mechanismName: String = saslServer.getMechanismName
+ AuthBridgeServer.userAuthMechanism.set(mechanismName)
+ if (AuthMethod.PLAIN.getMechanismName.equalsIgnoreCase(mechanismName)) {
+ AuthBridgeServer.remoteUser.set(endUser)
+ return wrapped.process(inProt, outProt)
+ }
+ AuthBridgeServer.authenticationMethod.set(UserGroupInformation.AuthenticationMethod.KERBEROS)
+ if (AuthMethod.TOKEN.getMechanismName.equalsIgnoreCase(mechanismName)) {
+ try {
+ val tokenId = SaslRpcServer.getIdentifier(authId, secretManager)
+ endUser = tokenId.getUser.getUserName
+ AuthBridgeServer.authenticationMethod.set(UserGroupInformation.AuthenticationMethod.TOKEN)
+ } catch {
+ case e: InvalidToken => throw new TException(e.getMessage)
+ }
+ }
+ var clientUgi: UserGroupInformation = null
+ try {
+ if (useProxy) {
+ clientUgi = UserGroupInformation.createProxyUser(
+ endUser, UserGroupInformation.getLoginUser)
+ AuthBridgeServer.remoteUser.set(clientUgi.getShortUserName)
+ debug(s"Set remoteUser : ${AuthBridgeServer.remoteUser.get}")
+ clientUgi.doAs(new PrivilegedExceptionAction[Boolean]() {
+ override def run: Boolean = try {
+ wrapped.process(inProt, outProt)
+ } catch {
+ case te: TException => throw new RuntimeException(te)
+ }
+ })
+ } else {
+ // use the short user name for the request
+ val endUserUgi: UserGroupInformation = UserGroupInformation.createRemoteUser(endUser)
+ AuthBridgeServer.remoteUser.set(endUserUgi.getShortUserName)
+ debug(s"Set remoteUser: ${AuthBridgeServer.remoteUser.get}, from endUser :" + endUser)
+ wrapped.process(inProt, outProt)
+ }
+ } catch {
+ case rte: RuntimeException if rte.getCause.isInstanceOf[TException] => throw rte.getCause
+ case rte: RuntimeException => throw rte
+ case ie: InterruptedException => throw new RuntimeException(ie) // unexpected!
+ case ioe: IOException => throw new RuntimeException(ioe)
+ } finally {
+ if (clientUgi != null) {
+ try {
+ FileSystem.closeAllForUGI(clientUgi)
+ } catch {
+ case exception: IOException =>
+ error(s"Could not clean up file-system handles for UGI: $clientUgi", exception)
+ }
+ }
+ }
+ }
+}
+
+object AuthBridgeServer extends Logging {
+ private[auth] val remoteAddress: ThreadLocal[InetAddress] = new ThreadLocal[InetAddress]() {
+ override protected def initialValue: InetAddress = null
+ }
+ private[auth] val authenticationMethod: ThreadLocal[AuthenticationMethod] =
+ new ThreadLocal[AuthenticationMethod]() {
+ override protected def initialValue: AuthenticationMethod = AuthenticationMethod.TOKEN
+ }
+ private[auth] val remoteUser: ThreadLocal[String] = new ThreadLocal[String]() {
+ override protected def initialValue: String = null
+ }
+ private[auth] val userAuthMechanism: ThreadLocal[String] = new ThreadLocal[String]() {
+ override protected def initialValue: String = AuthMethod.KERBEROS.getMechanismName
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthFactory.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthFactory.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthFactory.scala
new file mode 100644
index 0000000..6ac61d2
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthFactory.scala
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.auth
+
+import java.io.IOException
+import java.util
+import javax.security.auth.callback._
+import javax.security.auth.login.LoginException
+import javax.security.sasl.{AuthorizeCallback, Sasl}
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod
+import org.apache.hive.service.auth.{SaslQOP, TSetIpAddressProcessor}
+import org.apache.hive.service.auth.AuthenticationProviderFactory.AuthMethods
+import org.apache.hive.service.auth.HiveAuthConstants.AuthTypes
+import org.apache.hive.service.cli.HiveSQLException
+import org.apache.hive.service.rpc.thrift.TCLIService
+import org.apache.hive.service.rpc.thrift.TCLIService.Iface
+import org.apache.thrift.{TProcessor, TProcessorFactory}
+import org.apache.thrift.transport.{TTransport, TTransportException, TTransportFactory}
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.thriftserver.cli.ThriftCLIService
+
+/**
+ * This class is a porting of the parts we use from `HiveAuthFactory` by Hive.
+ */
+class AuthFactory(val conf: LivyConf) extends Logging {
+
+ private val authTypeStr = conf.get(LivyConf.THRIFT_AUTHENTICATION)
+ // ShimLoader.getHadoopShims().isSecurityEnabled() will only check that
+ // hadoopAuth is not simple, it does not guarantee it is kerberos
+ private val hadoopAuth = new Configuration().get(HADOOP_SECURITY_AUTHENTICATION)
+
+ private val secretManager = if (isSASLWithKerberizedHadoop) {
+ val sm = new LivyDelegationTokenSecretManager(conf)
+ try {
+ sm.startThreads()
+ } catch {
+ case e: IOException =>
+ throw new TTransportException("Failed to start token manager", e)
+ }
+ Some(sm)
+ } else {
+ None
+ }
+
+ private val saslServer: Option[AuthBridgeServer] = secretManager.map { sm =>
+ new AuthBridgeServer(sm)
+ }
+
+ def getSaslProperties: util.Map[String, String] = {
+ val saslProps = new util.HashMap[String, String]
+ val saslQOP = SaslQOP.fromString(conf.get(LivyConf.THRIFT_SASL_QOP))
+ saslProps.put(Sasl.QOP, saslQOP.toString)
+ saslProps.put(Sasl.SERVER_AUTH, "true")
+ saslProps
+ }
+
+ @throws[LoginException]
+ def getAuthTransFactory: TTransportFactory = {
+ val isAuthKerberos = authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName)
+ val isAuthNoSASL = authTypeStr.equalsIgnoreCase(AuthTypes.NOSASL.getAuthName)
+ // TODO: add LDAP and PAM when supported
+ val isAuthOther = authTypeStr.equalsIgnoreCase(AuthTypes.NONE.getAuthName) ||
+ authTypeStr.equalsIgnoreCase(AuthTypes.CUSTOM.getAuthName)
+
+ saslServer.map { server =>
+ val serverTransportFactory = try {
+ server.createSaslServerTransportFactory(getSaslProperties)
+ } catch {
+ case e: TTransportException =>
+ throw new LoginException(e.getMessage)
+ }
+ if (isAuthOther) {
+ PlainSaslServer.addPlainServerDefinition(serverTransportFactory, authTypeStr, conf)
+ } else if (!isAuthKerberos) {
+ throw new LoginException(s"Unsupported authentication type $authTypeStr")
+ }
+ server.wrapTransportFactory(serverTransportFactory)
+ }.getOrElse {
+ if (isAuthOther) {
+ PlainSaslServer.getPlainTransportFactory(authTypeStr, conf)
+ } else if (isAuthNoSASL) {
+ new TTransportFactory
+ } else {
+ throw new LoginException(s"Unsupported authentication type $authTypeStr")
+ }
+ }
+ }
+
+ /**
+ * Returns the thrift processor factory for binary mode
+ */
+ @throws[LoginException]
+ def getAuthProcFactory(service: ThriftCLIService): TProcessorFactory = {
+ if (saslServer.isDefined) {
+ new CLIServiceProcessorFactory(service, saslServer.get)
+ } else {
+ new SQLPlainProcessorFactory(service)
+ }
+ }
+
+ def getRemoteUser: String = saslServer.map(_.getRemoteUser).orNull
+
+ def getIpAddress: String =
+ saslServer.flatMap(s => Option(s.getRemoteAddress)).map(_.getHostAddress).orNull
+
+ def getUserAuthMechanism: String = saslServer.map(_.getUserAuthMechanism).orNull
+
+ def isSASLWithKerberizedHadoop: Boolean = {
+ "kerberos".equalsIgnoreCase(hadoopAuth) &&
+ !authTypeStr.equalsIgnoreCase(AuthTypes.NOSASL.getAuthName)
+ }
+
+ def isSASLKerberosUser: Boolean = {
+ AuthMethod.KERBEROS.getMechanismName == getUserAuthMechanism ||
+ AuthMethod.TOKEN.getMechanismName == getUserAuthMechanism
+ }
+
+ @throws[HiveSQLException]
+ def verifyDelegationToken(delegationToken: String): String = {
+ if (secretManager.isEmpty) {
+ throw new HiveSQLException(
+ "Delegation token only supported over kerberos authentication", "08S01")
+ }
+ try {
+ secretManager.get.verifyDelegationToken(delegationToken)
+ } catch {
+ case e: IOException =>
+ val msg = s"Error verifying delegation token $delegationToken"
+ error(msg, e)
+ throw new HiveSQLException(msg, "08S01", e)
+ }
+ }
+}
+
+class SQLPlainProcessorFactory(val service: Iface) extends TProcessorFactory(null) {
+
+ override def getProcessor(trans: TTransport): TProcessor = {
+ new TSetIpAddressProcessor[Iface](service)
+ }
+}
+
+class CLIServiceProcessorFactory(val service: Iface, val saslServer: AuthBridgeServer)
+ extends TProcessorFactory(null) {
+
+ override def getProcessor(trans: TTransport): TProcessor = {
+ val sqlProcessor = new TCLIService.Processor[Iface](service)
+ saslServer.wrapNonAssumingProcessor(sqlProcessor)
+ }
+}
+
+/**
+ * This is copied from Hive because its constructor is not accessible.
+ */
+class PlainServerCallbackHandler(authMethodStr: String, livyConf: LivyConf)
+ extends CallbackHandler {
+
+ private val authMethod: AuthMethods = AuthMethods.getValidAuthMethod(authMethodStr)
+
+ override def handle(callbacks: Array[Callback]): Unit = {
+ var username: String = null
+ var password: String = null
+ var ac: AuthorizeCallback = null
+
+ callbacks.foreach {
+ case nc: NameCallback => username = nc.getName
+ case pc: PasswordCallback => password = new String(pc.getPassword)
+ case c: AuthorizeCallback => ac = c
+ case other => throw new UnsupportedCallbackException(other)
+ }
+ val provider =
+ AuthenticationProvider.getAuthenticationProvider(authMethod.getAuthMethod, livyConf)
+ provider.Authenticate(username, password)
+ if (ac != null) {
+ ac.setAuthorized(true)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthenticationProvider.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthenticationProvider.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthenticationProvider.scala
new file mode 100644
index 0000000..9464af5
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/AuthenticationProvider.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.auth
+
+import java.lang.reflect.InvocationTargetException
+import javax.security.sasl.AuthenticationException
+
+import org.apache.hive.service.auth.PasswdAuthenticationProvider
+
+import org.apache.livy.LivyConf
+
+object AuthenticationProvider {
+ // TODO: support LDAP and PAM
+ val AUTH_METHODS = Seq("NONE", "CUSTOM")
+
+ @throws[AuthenticationException]
+ def getAuthenticationProvider(method: String, conf: LivyConf): PasswdAuthenticationProvider = {
+ method match {
+ case "NONE" => new NoneAuthenticationProvider
+ case "CUSTOM" => new CustomAuthenticationProvider(conf)
+ case _ => throw new AuthenticationException("Unsupported authentication method")
+ }
+ }
+}
+
+/**
+ * An implementation of [[PasswdAuthenticationProvider]] doing nothing.
+ */
+class NoneAuthenticationProvider extends PasswdAuthenticationProvider {
+ override def Authenticate(user: String, password: String): Unit = {
+ // Do nothing.
+ }
+}
+
+/**
+ * An implementation of [[PasswdAuthenticationProvider]] delegating the class configured in
+ * [[LivyConf.THRIFT_CUSTOM_AUTHENTICATION_CLASS]] to authenticate a user.
+ */
+class CustomAuthenticationProvider(conf: LivyConf) extends PasswdAuthenticationProvider {
+ private val customClass: Class[_ <: PasswdAuthenticationProvider] = {
+ Class.forName(conf.get(LivyConf.THRIFT_CUSTOM_AUTHENTICATION_CLASS))
+ .asSubclass(classOf[PasswdAuthenticationProvider])
+ }
+ val provider: PasswdAuthenticationProvider = {
+ // Try first a constructor with the LivyConf as parameter, then a constructor with no parameter
+ // of none of them is available this fails with an exception.
+ try {
+ customClass.getConstructor(classOf[LivyConf]).newInstance(conf)
+ } catch {
+ case _: NoSuchMethodException | _: InstantiationException | _: IllegalAccessException |
+ _: InvocationTargetException =>
+ customClass.getConstructor().newInstance()
+ }
+ }
+
+ override def Authenticate(user: String, password: String): Unit = {
+ provider.Authenticate(user, password)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/LivyDelegationTokenSecretManager.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/LivyDelegationTokenSecretManager.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/LivyDelegationTokenSecretManager.scala
new file mode 100644
index 0000000..e34306e
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/LivyDelegationTokenSecretManager.scala
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.auth
+
+import java.io.{ByteArrayInputStream, DataInputStream, IOException}
+
+import org.apache.hadoop.io.Text
+import org.apache.hadoop.security.token.Token
+import org.apache.hadoop.security.token.delegation.{AbstractDelegationTokenIdentifier, AbstractDelegationTokenSecretManager}
+
+import org.apache.livy.LivyConf
+
+/**
+ * A secret manager. It is taken from analogous implementation in the MapReduce client.
+ */
+class LivyDelegationTokenSecretManager(val livyConf: LivyConf)
+ extends AbstractDelegationTokenSecretManager[LivyDelegationTokenIdentifier](
+ livyConf.getTimeAsMs(LivyConf.THRIFT_DELEGATION_KEY_UPDATE_INTERVAL),
+ livyConf.getTimeAsMs(LivyConf.THRIFT_DELEGATION_TOKEN_MAX_LIFETIME),
+ livyConf.getTimeAsMs(LivyConf.THRIFT_DELEGATION_TOKEN_RENEW_INTERVAL),
+ livyConf.getTimeAsMs(LivyConf.THRIFT_DELEGATION_TOKEN_GC_INTERVAL)) {
+
+ override def createIdentifier: LivyDelegationTokenIdentifier = new LivyDelegationTokenIdentifier
+
+ /**
+ * Verify token string
+ */
+ @throws[IOException]
+ def verifyDelegationToken(tokenStrForm: String): String = {
+ val t = new Token[LivyDelegationTokenIdentifier]
+ t.decodeFromUrlString(tokenStrForm)
+ val id = getTokenIdentifier(t)
+ verifyToken(id, t.getPassword)
+ id.getUser.getShortUserName
+ }
+
+ @throws[IOException]
+ protected def getTokenIdentifier(
+ token: Token[LivyDelegationTokenIdentifier]): LivyDelegationTokenIdentifier = {
+ // turn bytes back into identifier for cache lookup
+ val buf = new ByteArrayInputStream(token.getIdentifier)
+ val in = new DataInputStream(buf)
+ val id = createIdentifier
+ id.readFields(in)
+ id
+ }
+}
+
+/**
+ * A delegation token identifier.
+ *
+ * @param owner the effective username of the token owner
+ * @param renewer the username of the renewer
+ * @param realUser the real username of the token owne
+ */
+class LivyDelegationTokenIdentifier(owner: Text, renewer: Text, realUser: Text)
+ extends AbstractDelegationTokenIdentifier(owner, renewer, realUser) {
+
+ def this() = this(new Text(), new Text(), new Text())
+
+ override def getKind: Text = LivyDelegationTokenIdentifier.LIVY_DELEGATION_KIND
+}
+
+object LivyDelegationTokenIdentifier {
+ val LIVY_DELEGATION_KIND = new Text("LIVY_DELEGATION_TOKEN")
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/PlainSaslServer.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/PlainSaslServer.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/PlainSaslServer.scala
new file mode 100644
index 0000000..ea8bd51
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/PlainSaslServer.scala
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.thriftserver.auth
+
+
+import java.io.IOException
+import java.security.{Provider, Security}
+import java.util
+import javax.security.auth.callback.Callback
+import javax.security.auth.callback.CallbackHandler
+import javax.security.auth.callback.NameCallback
+import javax.security.auth.callback.PasswordCallback
+import javax.security.auth.callback.UnsupportedCallbackException
+import javax.security.auth.login.LoginException
+import javax.security.sasl._
+
+import org.apache.hive.service.auth.AuthenticationProviderFactory.AuthMethods
+import org.apache.thrift.transport.TSaslServerTransport
+
+import org.apache.livy.LivyConf
+
+
+/**
+ * Sun JDK only provides a PLAIN client and no server. This class implements the Plain SASL server
+ * conforming to RFC #4616 (http://www.ietf.org/rfc/rfc4616.txt).
+ */
+class PlainSaslServer private[auth] (
+ val handler: CallbackHandler,
+ val authMethodStr: String) extends SaslServer {
+
+ AuthMethods.getValidAuthMethod(authMethodStr)
+
+ private var user: String = null
+ override def getMechanismName: String = PlainSaslServer.PLAIN_METHOD
+
+ @throws[SaslException]
+ override def evaluateResponse(response: Array[Byte]): Array[Byte] = {
+ try {
+ // parse the response
+ // message = [authzid] UTF8NUL authcid UTF8NUL passwd'
+ val tokenList: util.Deque[String] = new util.ArrayDeque[String]
+ var messageToken = new StringBuilder
+ for (b <- response) {
+ if (b == 0) {
+ tokenList.addLast(messageToken.toString)
+ messageToken = new StringBuilder
+ } else {
+ messageToken.append(b.toChar)
+ }
+ }
+ tokenList.addLast(messageToken.toString)
+ // validate response
+ if (tokenList.size < 2 || tokenList.size > 3) {
+ throw new SaslException("Invalid message format")
+ }
+ val passwd: String = tokenList.removeLast()
+ user = tokenList.removeLast()
+ // optional authzid
+ var authzId: String = null
+ if (tokenList.isEmpty) {
+ authzId = user
+ } else {
+ authzId = tokenList.removeLast()
+ }
+ if (user == null || user.isEmpty) {
+ throw new SaslException("No user name provided")
+ }
+ if (passwd == null || passwd.isEmpty) {
+ throw new SaslException("No password name provided")
+ }
+ val nameCallback = new NameCallback("User")
+ nameCallback.setName(user)
+ val pcCallback = new PasswordCallback("Password", false)
+ pcCallback.setPassword(passwd.toCharArray)
+ val acCallback = new AuthorizeCallback(user, authzId)
+ val cbList = Array[Callback](nameCallback, pcCallback, acCallback)
+ handler.handle(cbList)
+ if (!acCallback.isAuthorized) {
+ throw new SaslException("Authentication failed")
+ }
+ } catch {
+ case eL: IllegalStateException =>
+ throw new SaslException("Invalid message format", eL)
+ case eI: IOException =>
+ throw new SaslException("Error validating the login", eI)
+ case eU: UnsupportedCallbackException =>
+ throw new SaslException("Error validating the login", eU)
+ }
+ null
+ }
+
+ override def isComplete: Boolean = user != null
+
+ override def getAuthorizationID: String = user
+
+ override def unwrap(incoming: Array[Byte], offset: Int, len: Int): Array[Byte] = {
+ throw new UnsupportedOperationException
+ }
+
+ override def wrap(outgoing: Array[Byte], offset: Int, len: Int): Array[Byte] = {
+ throw new UnsupportedOperationException
+ }
+
+ override def getNegotiatedProperty(propName: String): Object = null
+
+ override def dispose(): Unit = {}
+}
+
+object PlainSaslServer {
+ val PLAIN_METHOD = "PLAIN"
+
+ Security.addProvider(new SaslPlainProvider)
+
+ def getPlainTransportFactory(
+ authTypeStr: String,
+ conf: LivyConf): TSaslServerTransport.Factory = {
+ val saslFactory = new TSaslServerTransport.Factory()
+ addPlainServerDefinition(saslFactory, authTypeStr, conf)
+ saslFactory
+ }
+
+ def addPlainServerDefinition(
+ saslFactory: TSaslServerTransport.Factory,
+ authTypeStr: String,
+ conf: LivyConf): Unit = {
+ try {
+ saslFactory.addServerDefinition("PLAIN",
+ authTypeStr,
+ null,
+ new util.HashMap[String, String](),
+ new PlainServerCallbackHandler(authTypeStr, conf))
+ } catch {
+ case e: AuthenticationException =>
+ throw new LoginException(s"Error setting callback handler $e")
+ }
+ }
+}
+
+class SaslPlainServerFactory extends SaslServerFactory {
+ override def createSaslServer(
+ mechanism: String,
+ protocol: String,
+ serverName: String,
+ props: util.Map[String, _],
+ cbh: CallbackHandler): PlainSaslServer = {
+ if (PlainSaslServer.PLAIN_METHOD == mechanism) {
+ try {
+ new PlainSaslServer(cbh, protocol)
+ } catch {
+ case _: SaslException =>
+ /* This is to fulfill the contract of the interface which states that an exception shall
+ be thrown when a SaslServer cannot be created due to an error but null should be
+ returned when a Server can't be created due to the parameters supplied. And the only
+ thing PlainSaslServer can fail on is a non-supported authentication mechanism.
+ That's why we return null instead of throwing the Exception */
+ null
+ }
+ } else {
+ null
+ }
+ }
+
+ override def getMechanismNames(props: util.Map[String, _]): Array[String] = {
+ Array[String](PlainSaslServer.PLAIN_METHOD)
+ }
+}
+
+class SaslPlainProvider extends Provider("LivySaslPlain", 1.0, "Livy Plain SASL provider") {
+ put("SaslServerFactory.PLAIN", classOf[SaslPlainServerFactory].getName)
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/47d3ee6b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThreadPoolExecutorWithOomHook.scala
----------------------------------------------------------------------
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThreadPoolExecutorWithOomHook.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThreadPoolExecutorWithOomHook.scala
new file mode 100644
index 0000000..0a06baa
--- /dev/null
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThreadPoolExecutorWithOomHook.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.thriftserver.cli
+
+import java.util.concurrent._
+
+/**
+ * This class is taken from Hive, because it is package private so it cannot be accessed.
+ * If it will become public we can remove this from here.
+ */
+class ThreadPoolExecutorWithOomHook(
+ corePoolSize: Int,
+ maximumPoolSize: Int,
+ keepAliveTime: Long,
+ unit: TimeUnit,
+ workQueue: BlockingQueue[Runnable],
+ threadFactory: ThreadFactory,
+ val oomHook: Runnable)
+ extends ThreadPoolExecutor(
+ corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory) {
+
+ override protected def afterExecute(r: Runnable, t: Throwable): Unit = {
+ super.afterExecute(r, t)
+ if (t == null && r.isInstanceOf[Future[_]] ) {
+ try {
+ val future: Future[_] = r.asInstanceOf[Future[_]]
+ if (future.isDone) {
+ future.get
+ }
+ } catch {
+ case _: InterruptedException => Thread.currentThread.interrupt()
+ case _: OutOfMemoryError => oomHook.run()
+ case _: Throwable => // Do nothing
+ }
+ } else if (t.isInstanceOf[OutOfMemoryError]) {
+ oomHook.run()
+ }
+ }
+}