You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by js...@apache.org on 2017/07/05 03:06:30 UTC
[01/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Repository: incubator-livy
Updated Branches:
refs/heads/master cb5b8aac5 -> 412ccc8fc
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/org/apache/livy/test/jobs/Failure.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/org/apache/livy/test/jobs/Failure.java b/test-lib/src/main/java/org/apache/livy/test/jobs/Failure.java
new file mode 100644
index 0000000..f56f400
--- /dev/null
+++ b/test-lib/src/main/java/org/apache/livy/test/jobs/Failure.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class Failure implements Job<Void> {
+
+ @Override
+ public Void call(JobContext jc) {
+ throw new JobFailureException();
+ }
+
+ public static class JobFailureException extends RuntimeException {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/org/apache/livy/test/jobs/FileReader.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/org/apache/livy/test/jobs/FileReader.java b/test-lib/src/main/java/org/apache/livy/test/jobs/FileReader.java
new file mode 100644
index 0000000..8b22c8f
--- /dev/null
+++ b/test-lib/src/main/java/org/apache/livy/test/jobs/FileReader.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import org.apache.spark.SparkFiles;
+import org.apache.spark.api.java.function.Function;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class FileReader implements Job<String> {
+
+ private final boolean isResource;
+ private final String fileName;
+
+ public FileReader(String fileName, boolean isResource) {
+ this.fileName = fileName;
+ this.isResource = isResource;
+ }
+
+ @Override
+ public String call(JobContext jc) {
+ return jc.sc().parallelize(Arrays.asList(1)).map(new Reader()).collect().get(0);
+ }
+
+ private class Reader implements Function<Integer, String> {
+
+ @Override
+ public String call(Integer i) throws Exception {
+ InputStream in;
+ if (isResource) {
+ ClassLoader ccl = Thread.currentThread().getContextClassLoader();
+ in = ccl.getResourceAsStream(fileName);
+ if (in == null) {
+ throw new IOException("Resource not found: " + fileName);
+ }
+ } else {
+ in = new FileInputStream(SparkFiles.get(fileName));
+ }
+ try {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buf = new byte[1024];
+ int read;
+ while ((read = in.read(buf)) >= 0) {
+ out.write(buf, 0, read);
+ }
+ byte[] bytes = out.toByteArray();
+ return new String(bytes, 0, bytes.length, UTF_8);
+ } finally {
+ in.close();
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/org/apache/livy/test/jobs/GetCurrentUser.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/org/apache/livy/test/jobs/GetCurrentUser.java b/test-lib/src/main/java/org/apache/livy/test/jobs/GetCurrentUser.java
new file mode 100644
index 0000000..28a6ddc
--- /dev/null
+++ b/test-lib/src/main/java/org/apache/livy/test/jobs/GetCurrentUser.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs;
+
+import org.apache.hadoop.security.UserGroupInformation;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class GetCurrentUser implements Job<String> {
+
+ @Override
+ public String call(JobContext jc) throws Exception {
+ return UserGroupInformation.getCurrentUser().getUserName();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/org/apache/livy/test/jobs/SQLGetTweets.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/org/apache/livy/test/jobs/SQLGetTweets.java b/test-lib/src/main/java/org/apache/livy/test/jobs/SQLGetTweets.java
new file mode 100644
index 0000000..a17f188
--- /dev/null
+++ b/test-lib/src/main/java/org/apache/livy/test/jobs/SQLGetTweets.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs;
+
+import java.io.File;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SQLContext;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class SQLGetTweets implements Job<List<String>> {
+
+ private final boolean useHiveContext;
+
+ public SQLGetTweets(boolean useHiveContext) {
+ this.useHiveContext = useHiveContext;
+ }
+
+ @Override
+ public List<String> call(JobContext jc) throws Exception {
+ InputStream source = getClass().getResourceAsStream("/testweet.json");
+
+ // Save the resource as a file in HDFS (or the local tmp dir when using a local filesystem).
+ URI input;
+ File local = File.createTempFile("tweets", ".json", jc.getLocalTmpDir());
+ Files.copy(source, local.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ FileSystem fs = FileSystem.get(jc.sc().sc().hadoopConfiguration());
+ if ("file".equals(fs.getUri().getScheme())) {
+ input = local.toURI();
+ } else {
+ String uuid = UUID.randomUUID().toString();
+ Path target = new Path("/tmp/" + uuid + "-tweets.json");
+ fs.copyFromLocalFile(new Path(local.toURI()), target);
+ input = target.toUri();
+ }
+
+ SQLContext sqlctx = useHiveContext ? jc.hivectx() : jc.sqlctx();
+ sqlctx.jsonFile(input.toString()).registerTempTable("tweets");
+
+ List<String> tweetList = new ArrayList<>();
+ Row[] result =
+ (Row[])(sqlctx.sql("SELECT text, retweetCount FROM tweets ORDER BY retweetCount LIMIT 10")
+ .collect());
+ for (Row r : result) {
+ tweetList.add(r.toString());
+ }
+ return tweetList;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/org/apache/livy/test/jobs/Sleeper.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/org/apache/livy/test/jobs/Sleeper.java b/test-lib/src/main/java/org/apache/livy/test/jobs/Sleeper.java
new file mode 100644
index 0000000..89fe785
--- /dev/null
+++ b/test-lib/src/main/java/org/apache/livy/test/jobs/Sleeper.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class Sleeper implements Job<Void> {
+
+ private final long millis;
+
+ public Sleeper(long millis) {
+ this.millis = millis;
+ }
+
+ @Override
+ public Void call(JobContext jc) throws Exception {
+ Thread.sleep(millis);
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/org/apache/livy/test/jobs/SmallCount.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/org/apache/livy/test/jobs/SmallCount.java b/test-lib/src/main/java/org/apache/livy/test/jobs/SmallCount.java
new file mode 100644
index 0000000..e7c8ee4
--- /dev/null
+++ b/test-lib/src/main/java/org/apache/livy/test/jobs/SmallCount.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class SmallCount implements Job<Long> {
+
+ private final int count;
+
+ public SmallCount(int count) {
+ this.count = count;
+ }
+
+ @Override
+ public Long call(JobContext jc) {
+ Random r = new Random();
+ int partitions = Math.min(r.nextInt(10) + 1, count);
+
+ List<Integer> elements = new ArrayList<>(count);
+ for (int i = 0; i < count; i++) {
+ elements.add(r.nextInt());
+ }
+
+ return jc.sc().parallelize(elements, partitions).count();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/org/apache/livy/test/jobs/VoidJob.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/org/apache/livy/test/jobs/VoidJob.java b/test-lib/src/main/java/org/apache/livy/test/jobs/VoidJob.java
new file mode 100644
index 0000000..d6875c8
--- /dev/null
+++ b/test-lib/src/main/java/org/apache/livy/test/jobs/VoidJob.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class VoidJob implements Job<Void> {
+ @Override
+ public Void call(JobContext jc) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/scala/com/cloudera/livy/test/jobs/ScalaEcho.scala
----------------------------------------------------------------------
diff --git a/test-lib/src/main/scala/com/cloudera/livy/test/jobs/ScalaEcho.scala b/test-lib/src/main/scala/com/cloudera/livy/test/jobs/ScalaEcho.scala
deleted file mode 100644
index e504d69..0000000
--- a/test-lib/src/main/scala/com/cloudera/livy/test/jobs/ScalaEcho.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs
-
-import scala.reflect.ClassTag
-
-import com.cloudera.livy.{Job, JobContext}
-
-case class ValueHolder[T](value: T)
-
-class ScalaEcho[T: ClassTag](val value: T)(implicit val tag: ClassTag[T]) extends Job[T] {
-
- override def call(jc: JobContext): T = {
- jc.sc().sc.parallelize(Seq(value), 1)(tag).collect()(0)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/scala/org/apache/livy/test/jobs/ScalaEcho.scala
----------------------------------------------------------------------
diff --git a/test-lib/src/main/scala/org/apache/livy/test/jobs/ScalaEcho.scala b/test-lib/src/main/scala/org/apache/livy/test/jobs/ScalaEcho.scala
new file mode 100644
index 0000000..a4c6032
--- /dev/null
+++ b/test-lib/src/main/scala/org/apache/livy/test/jobs/ScalaEcho.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs
+
+import scala.reflect.ClassTag
+
+import org.apache.livy.{Job, JobContext}
+
+case class ValueHolder[T](value: T)
+
+class ScalaEcho[T: ClassTag](val value: T)(implicit val tag: ClassTag[T]) extends Job[T] {
+
+ override def call(jc: JobContext): T = {
+ jc.sc().sc.parallelize(Seq(value), 1)(tag).collect()(0)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/spark2/java/com/cloudera/livy/test/jobs/spark2/DatasetTest.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/spark2/java/com/cloudera/livy/test/jobs/spark2/DatasetTest.java b/test-lib/src/main/spark2/java/com/cloudera/livy/test/jobs/spark2/DatasetTest.java
deleted file mode 100644
index 4259d5e..0000000
--- a/test-lib/src/main/spark2/java/com/cloudera/livy/test/jobs/spark2/DatasetTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs.spark2;
-
-import java.util.Arrays;
-
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.FilterFunction;
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.types.DataTypes;
-import org.apache.spark.sql.types.StructField;
-import org.apache.spark.sql.types.StructType;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class DatasetTest implements Job<Long> {
-
- @Override
- public Long call(JobContext jc) throws Exception {
- SparkSession spark = jc.sparkSession();
-
- JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
- JavaRDD<Row> rdd = sc.parallelize(Arrays.asList(1, 2, 3)).map(
- new Function<Integer, Row>() {
- public Row call(Integer integer) throws Exception {
- return RowFactory.create(integer);
- }
- });
- StructType schema = DataTypes.createStructType(new StructField[] {
- DataTypes.createStructField("value", DataTypes.IntegerType, false)
- });
-
- Dataset<Row> ds = spark.createDataFrame(rdd, schema);
-
- return ds.filter(new FilterFunction<Row>() {
- @Override
- public boolean call(Row row) throws Exception {
- return row.getInt(0) >= 2;
- }
- }).count();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/spark2/java/com/cloudera/livy/test/jobs/spark2/SparkSessionTest.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/spark2/java/com/cloudera/livy/test/jobs/spark2/SparkSessionTest.java b/test-lib/src/main/spark2/java/com/cloudera/livy/test/jobs/spark2/SparkSessionTest.java
deleted file mode 100644
index 1019670..0000000
--- a/test-lib/src/main/spark2/java/com/cloudera/livy/test/jobs/spark2/SparkSessionTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs.spark2;
-
-import java.util.Arrays;
-
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SparkSession;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class SparkSessionTest implements Job<Long> {
-
- @Override
- public Long call(JobContext jc) throws Exception {
- // Make sure SparkSession and SparkContext is callable
- SparkSession session = jc.sparkSession();
-
- JavaSparkContext sc = new JavaSparkContext(session.sparkContext());
- return sc.parallelize(Arrays.asList(1, 2, 3)).count();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/spark2/java/org/apache/livy/test/jobs/spark2/DatasetTest.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/spark2/java/org/apache/livy/test/jobs/spark2/DatasetTest.java b/test-lib/src/main/spark2/java/org/apache/livy/test/jobs/spark2/DatasetTest.java
new file mode 100644
index 0000000..92940c5
--- /dev/null
+++ b/test-lib/src/main/spark2/java/org/apache/livy/test/jobs/spark2/DatasetTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs.spark2;
+
+import java.util.Arrays;
+
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.FilterFunction;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class DatasetTest implements Job<Long> {
+
+ @Override
+ public Long call(JobContext jc) throws Exception {
+ SparkSession spark = jc.sparkSession();
+
+ JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
+ JavaRDD<Row> rdd = sc.parallelize(Arrays.asList(1, 2, 3)).map(
+ new Function<Integer, Row>() {
+ public Row call(Integer integer) throws Exception {
+ return RowFactory.create(integer);
+ }
+ });
+ StructType schema = DataTypes.createStructType(new StructField[] {
+ DataTypes.createStructField("value", DataTypes.IntegerType, false)
+ });
+
+ Dataset<Row> ds = spark.createDataFrame(rdd, schema);
+
+ return ds.filter(new FilterFunction<Row>() {
+ @Override
+ public boolean call(Row row) throws Exception {
+ return row.getInt(0) >= 2;
+ }
+ }).count();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/spark2/java/org/apache/livy/test/jobs/spark2/SparkSessionTest.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/spark2/java/org/apache/livy/test/jobs/spark2/SparkSessionTest.java b/test-lib/src/main/spark2/java/org/apache/livy/test/jobs/spark2/SparkSessionTest.java
new file mode 100644
index 0000000..d04d5e5
--- /dev/null
+++ b/test-lib/src/main/spark2/java/org/apache/livy/test/jobs/spark2/SparkSessionTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs.spark2;
+
+import java.util.Arrays;
+
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SparkSession;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class SparkSessionTest implements Job<Long> {
+
+ @Override
+ public Long call(JobContext jc) throws Exception {
+ // Make sure SparkSession and SparkContext is callable
+ SparkSession session = jc.sparkSession();
+
+ JavaSparkContext sc = new JavaSparkContext(session.sparkContext());
+ return sc.parallelize(Arrays.asList(1, 2, 3)).count();
+ }
+}
[12/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/bootstrap.min.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/bootstrap.min.js b/server/src/main/resources/org/apache/livy/server/ui/static/bootstrap.min.js
new file mode 100644
index 0000000..9bcd2fc
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/bootstrap.min.js
@@ -0,0 +1,7 @@
+/*!
+ * Bootstrap v3.3.7 (http://getbootstrap.com)
+ * Copyright 2011-2016 Twitter, Inc.
+ * Licensed under the MIT license
+ */
+if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");+function(a){"use strict";var b=a.fn.jquery.split(" ")[0].split(".");if(b[0]<2&&b[1]<9||1==b[0]&&9==b[1]&&b[2]<1||b[0]>3)throw new Error("Bootstrap's JavaScript requires jQuery version 1.9.1 or higher, but lower than version 4")}(jQuery),+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var c in b)if(void 0!==a.style[c])return{end:b[c]};return!1}a.fn.emulateTransitionEnd=function(b){var c=!1,d=this;a(this).one("bsTransitionEnd",function(){c=!0});var e=function(){c||a(d).trigger(a.support.transition.end)};return setTimeout(e,b),this},a(function(){a.support.transition=b(),a.support.transition&&(a.event.special.bsTransitionEnd={bindType:a.support.transition.end,delegateType:a.support.transition.end,handle:function(b){if(a(b
.target).is(this))return b.handleObj.handler.apply(this,arguments)}})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var c=a(this),e=c.data("bs.alert");e||c.data("bs.alert",e=new d(this)),"string"==typeof b&&e[b].call(c)})}var c='[data-dismiss="alert"]',d=function(b){a(b).on("click",c,this.close)};d.VERSION="3.3.7",d.TRANSITION_DURATION=150,d.prototype.close=function(b){function c(){g.detach().trigger("closed.bs.alert").remove()}var e=a(this),f=e.attr("data-target");f||(f=e.attr("href"),f=f&&f.replace(/.*(?=#[^\s]*$)/,""));var g=a("#"===f?[]:f);b&&b.preventDefault(),g.length||(g=e.closest(".alert")),g.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(g.removeClass("in"),a.support.transition&&g.hasClass("fade")?g.one("bsTransitionEnd",c).emulateTransitionEnd(d.TRANSITION_DURATION):c())};var e=a.fn.alert;a.fn.alert=b,a.fn.alert.Constructor=d,a.fn.alert.noConflict=function(){return a.fn.alert=e,this},a(document).on("click.bs.alert.data-api",c
,d.prototype.close)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.button"),f="object"==typeof b&&b;e||d.data("bs.button",e=new c(this,f)),"toggle"==b?e.toggle():b&&e.setState(b)})}var c=function(b,d){this.$element=a(b),this.options=a.extend({},c.DEFAULTS,d),this.isLoading=!1};c.VERSION="3.3.7",c.DEFAULTS={loadingText:"loading..."},c.prototype.setState=function(b){var c="disabled",d=this.$element,e=d.is("input")?"val":"html",f=d.data();b+="Text",null==f.resetText&&d.data("resetText",d[e]()),setTimeout(a.proxy(function(){d[e](null==f[b]?this.options[b]:f[b]),"loadingText"==b?(this.isLoading=!0,d.addClass(c).attr(c,c).prop(c,!0)):this.isLoading&&(this.isLoading=!1,d.removeClass(c).removeAttr(c).prop(c,!1))},this),0)},c.prototype.toggle=function(){var a=!0,b=this.$element.closest('[data-toggle="buttons"]');if(b.length){var c=this.$element.find("input");"radio"==c.prop("type")?(c.prop("checked")&&(a=!1),b.find(".active").removeCla
ss("active"),this.$element.addClass("active")):"checkbox"==c.prop("type")&&(c.prop("checked")!==this.$element.hasClass("active")&&(a=!1),this.$element.toggleClass("active")),c.prop("checked",this.$element.hasClass("active")),a&&c.trigger("change")}else this.$element.attr("aria-pressed",!this.$element.hasClass("active")),this.$element.toggleClass("active")};var d=a.fn.button;a.fn.button=b,a.fn.button.Constructor=c,a.fn.button.noConflict=function(){return a.fn.button=d,this},a(document).on("click.bs.button.data-api",'[data-toggle^="button"]',function(c){var d=a(c.target).closest(".btn");b.call(d,"toggle"),a(c.target).is('input[type="radio"], input[type="checkbox"]')||(c.preventDefault(),d.is("input,button")?d.trigger("focus"):d.find("input:visible,button:visible").first().trigger("focus"))}).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',function(b){a(b.target).closest(".btn").toggleClass("focus",/^focus(in)?$/.test(b.type))})}(jQuery),+function(a){"us
e strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.carousel"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof b&&b),g="string"==typeof b?b:f.slide;e||d.data("bs.carousel",e=new c(this,f)),"number"==typeof b?e.to(b):g?e[g]():f.interval&&e.pause().cycle()})}var c=function(b,c){this.$element=a(b),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=null,this.sliding=null,this.interval=null,this.$active=null,this.$items=null,this.options.keyboard&&this.$element.on("keydown.bs.carousel",a.proxy(this.keydown,this)),"hover"==this.options.pause&&!("ontouchstart"in document.documentElement)&&this.$element.on("mouseenter.bs.carousel",a.proxy(this.pause,this)).on("mouseleave.bs.carousel",a.proxy(this.cycle,this))};c.VERSION="3.3.7",c.TRANSITION_DURATION=600,c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0,keyboard:!0},c.prototype.keydown=function(a){if(!/input|textarea/i.test(a.target.tagName)){switch(a.which){case 37:this.prev();b
reak;case 39:this.next();break;default:return}a.preventDefault()}},c.prototype.cycle=function(b){return b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(a){return this.$items=a.parent().children(".item"),this.$items.index(a||this.$active)},c.prototype.getItemForDirection=function(a,b){var c=this.getItemIndex(b),d="prev"==a&&0===c||"next"==a&&c==this.$items.length-1;if(d&&!this.options.wrap)return b;var e="prev"==a?-1:1,f=(c+e)%this.$items.length;return this.$items.eq(f)},c.prototype.to=function(a){var b=this,c=this.getItemIndex(this.$active=this.$element.find(".item.active"));if(!(a>this.$items.length-1||a<0))return this.sliding?this.$element.one("slid.bs.carousel",function(){b.to(a)}):c==a?this.pause().cycle():this.slide(a>c?"next":"prev",this.$items.eq(a))},c.prototype.pause=function(b){return b||(this.paused=!0),thi
s.$element.find(".next, .prev").length&&a.support.transition&&(this.$element.trigger(a.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},c.prototype.next=function(){if(!this.sliding)return this.slide("next")},c.prototype.prev=function(){if(!this.sliding)return this.slide("prev")},c.prototype.slide=function(b,d){var e=this.$element.find(".item.active"),f=d||this.getItemForDirection(b,e),g=this.interval,h="next"==b?"left":"right",i=this;if(f.hasClass("active"))return this.sliding=!1;var j=f[0],k=a.Event("slide.bs.carousel",{relatedTarget:j,direction:h});if(this.$element.trigger(k),!k.isDefaultPrevented()){if(this.sliding=!0,g&&this.pause(),this.$indicators.length){this.$indicators.find(".active").removeClass("active");var l=a(this.$indicators.children()[this.getItemIndex(f)]);l&&l.addClass("active")}var m=a.Event("slid.bs.carousel",{relatedTarget:j,direction:h});return a.support.transition&&this.$element.hasClass("slide")?(f.addClass(b),f[0].offs
etWidth,e.addClass(h),f.addClass(h),e.one("bsTransitionEnd",function(){f.removeClass([b,h].join(" ")).addClass("active"),e.removeClass(["active",h].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger(m)},0)}).emulateTransitionEnd(c.TRANSITION_DURATION)):(e.removeClass("active"),f.addClass("active"),this.sliding=!1,this.$element.trigger(m)),g&&this.cycle(),this}};var d=a.fn.carousel;a.fn.carousel=b,a.fn.carousel.Constructor=c,a.fn.carousel.noConflict=function(){return a.fn.carousel=d,this};var e=function(c){var d,e=a(this),f=a(e.attr("data-target")||(d=e.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""));if(f.hasClass("carousel")){var g=a.extend({},f.data(),e.data()),h=e.attr("data-slide-to");h&&(g.interval=!1),b.call(f,g),h&&f.data("bs.carousel").to(h),c.preventDefault()}};a(document).on("click.bs.carousel.data-api","[data-slide]",e).on("click.bs.carousel.data-api","[data-slide-to]",e),a(window).on("load",function(){a('[data-ride="carousel"]').each(function(){var c=a(this);
b.call(c,c.data())})})}(jQuery),+function(a){"use strict";function b(b){var c,d=b.attr("data-target")||(c=b.attr("href"))&&c.replace(/.*(?=#[^\s]+$)/,"");return a(d)}function c(b){return this.each(function(){var c=a(this),e=c.data("bs.collapse"),f=a.extend({},d.DEFAULTS,c.data(),"object"==typeof b&&b);!e&&f.toggle&&/show|hide/.test(b)&&(f.toggle=!1),e||c.data("bs.collapse",e=new d(this,f)),"string"==typeof b&&e[b]()})}var d=function(b,c){this.$element=a(b),this.options=a.extend({},d.DEFAULTS,c),this.$trigger=a('[data-toggle="collapse"][href="#'+b.id+'"],[data-toggle="collapse"][data-target="#'+b.id+'"]'),this.transitioning=null,this.options.parent?this.$parent=this.getParent():this.addAriaAndCollapsedClass(this.$element,this.$trigger),this.options.toggle&&this.toggle()};d.VERSION="3.3.7",d.TRANSITION_DURATION=350,d.DEFAULTS={toggle:!0},d.prototype.dimension=function(){var a=this.$element.hasClass("width");return a?"width":"height"},d.prototype.show=function(){if(!this.transitioning&
&!this.$element.hasClass("in")){var b,e=this.$parent&&this.$parent.children(".panel").children(".in, .collapsing");if(!(e&&e.length&&(b=e.data("bs.collapse"),b&&b.transitioning))){var f=a.Event("show.bs.collapse");if(this.$element.trigger(f),!f.isDefaultPrevented()){e&&e.length&&(c.call(e,"hide"),b||e.data("bs.collapse",null));var g=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[g](0).attr("aria-expanded",!0),this.$trigger.removeClass("collapsed").attr("aria-expanded",!0),this.transitioning=1;var h=function(){this.$element.removeClass("collapsing").addClass("collapse in")[g](""),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return h.call(this);var i=a.camelCase(["scroll",g].join("-"));this.$element.one("bsTransitionEnd",a.proxy(h,this)).emulateTransitionEnd(d.TRANSITION_DURATION)[g](this.$element[0][i])}}}},d.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var b=a.Event("hide.b
s.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.dimension();this.$element[c](this.$element[c]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse in").attr("aria-expanded",!1),this.$trigger.addClass("collapsed").attr("aria-expanded",!1),this.transitioning=1;var e=function(){this.transitioning=0,this.$element.removeClass("collapsing").addClass("collapse").trigger("hidden.bs.collapse")};return a.support.transition?void this.$element[c](0).one("bsTransitionEnd",a.proxy(e,this)).emulateTransitionEnd(d.TRANSITION_DURATION):e.call(this)}}},d.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()},d.prototype.getParent=function(){return a(this.options.parent).find('[data-toggle="collapse"][data-parent="'+this.options.parent+'"]').each(a.proxy(function(c,d){var e=a(d);this.addAriaAndCollapsedClass(b(e),e)},this)).end()},d.prototype.addAriaAndCollapsedClass=function(a,b){var c=a.hasClass("in");a.attr("aria-expande
d",c),b.toggleClass("collapsed",!c).attr("aria-expanded",c)};var e=a.fn.collapse;a.fn.collapse=c,a.fn.collapse.Constructor=d,a.fn.collapse.noConflict=function(){return a.fn.collapse=e,this},a(document).on("click.bs.collapse.data-api",'[data-toggle="collapse"]',function(d){var e=a(this);e.attr("data-target")||d.preventDefault();var f=b(e),g=f.data("bs.collapse"),h=g?"toggle":e.data();c.call(f,h)})}(jQuery),+function(a){"use strict";function b(b){var c=b.attr("data-target");c||(c=b.attr("href"),c=c&&/#[A-Za-z]/.test(c)&&c.replace(/.*(?=#[^\s]*$)/,""));var d=c&&a(c);return d&&d.length?d:b.parent()}function c(c){c&&3===c.which||(a(e).remove(),a(f).each(function(){var d=a(this),e=b(d),f={relatedTarget:this};e.hasClass("open")&&(c&&"click"==c.type&&/input|textarea/i.test(c.target.tagName)&&a.contains(e[0],c.target)||(e.trigger(c=a.Event("hide.bs.dropdown",f)),c.isDefaultPrevented()||(d.attr("aria-expanded","false"),e.removeClass("open").trigger(a.Event("hidden.bs.dropdown",f)))))}))}funct
ion d(b){return this.each(function(){var c=a(this),d=c.data("bs.dropdown");d||c.data("bs.dropdown",d=new g(this)),"string"==typeof b&&d[b].call(c)})}var e=".dropdown-backdrop",f='[data-toggle="dropdown"]',g=function(b){a(b).on("click.bs.dropdown",this.toggle)};g.VERSION="3.3.7",g.prototype.toggle=function(d){var e=a(this);if(!e.is(".disabled, :disabled")){var f=b(e),g=f.hasClass("open");if(c(),!g){"ontouchstart"in document.documentElement&&!f.closest(".navbar-nav").length&&a(document.createElement("div")).addClass("dropdown-backdrop").insertAfter(a(this)).on("click",c);var h={relatedTarget:this};if(f.trigger(d=a.Event("show.bs.dropdown",h)),d.isDefaultPrevented())return;e.trigger("focus").attr("aria-expanded","true"),f.toggleClass("open").trigger(a.Event("shown.bs.dropdown",h))}return!1}},g.prototype.keydown=function(c){if(/(38|40|27|32)/.test(c.which)&&!/input|textarea/i.test(c.target.tagName)){var d=a(this);if(c.preventDefault(),c.stopPropagation(),!d.is(".disabled, :disabled")){v
ar e=b(d),g=e.hasClass("open");if(!g&&27!=c.which||g&&27==c.which)return 27==c.which&&e.find(f).trigger("focus"),d.trigger("click");var h=" li:not(.disabled):visible a",i=e.find(".dropdown-menu"+h);if(i.length){var j=i.index(c.target);38==c.which&&j>0&&j--,40==c.which&&j<i.length-1&&j++,~j||(j=0),i.eq(j).trigger("focus")}}}};var h=a.fn.dropdown;a.fn.dropdown=d,a.fn.dropdown.Constructor=g,a.fn.dropdown.noConflict=function(){return a.fn.dropdown=h,this},a(document).on("click.bs.dropdown.data-api",c).on("click.bs.dropdown.data-api",".dropdown form",function(a){a.stopPropagation()}).on("click.bs.dropdown.data-api",f,g.prototype.toggle).on("keydown.bs.dropdown.data-api",f,g.prototype.keydown).on("keydown.bs.dropdown.data-api",".dropdown-menu",g.prototype.keydown)}(jQuery),+function(a){"use strict";function b(b,d){return this.each(function(){var e=a(this),f=e.data("bs.modal"),g=a.extend({},c.DEFAULTS,e.data(),"object"==typeof b&&b);f||e.data("bs.modal",f=new c(this,g)),"string"==typeof b?
f[b](d):g.show&&f.show(d)})}var c=function(b,c){this.options=c,this.$body=a(document.body),this.$element=a(b),this.$dialog=this.$element.find(".modal-dialog"),this.$backdrop=null,this.isShown=null,this.originalBodyPad=null,this.scrollbarWidth=0,this.ignoreBackdropClick=!1,this.options.remote&&this.$element.find(".modal-content").load(this.options.remote,a.proxy(function(){this.$element.trigger("loaded.bs.modal")},this))};c.VERSION="3.3.7",c.TRANSITION_DURATION=300,c.BACKDROP_TRANSITION_DURATION=150,c.DEFAULTS={backdrop:!0,keyboard:!0,show:!0},c.prototype.toggle=function(a){return this.isShown?this.hide():this.show(a)},c.prototype.show=function(b){var d=this,e=a.Event("show.bs.modal",{relatedTarget:b});this.$element.trigger(e),this.isShown||e.isDefaultPrevented()||(this.isShown=!0,this.checkScrollbar(),this.setScrollbar(),this.$body.addClass("modal-open"),this.escape(),this.resize(),this.$element.on("click.dismiss.bs.modal",'[data-dismiss="modal"]',a.proxy(this.hide,this)),this.$dial
og.on("mousedown.dismiss.bs.modal",function(){d.$element.one("mouseup.dismiss.bs.modal",function(b){a(b.target).is(d.$element)&&(d.ignoreBackdropClick=!0)})}),this.backdrop(function(){var e=a.support.transition&&d.$element.hasClass("fade");d.$element.parent().length||d.$element.appendTo(d.$body),d.$element.show().scrollTop(0),d.adjustDialog(),e&&d.$element[0].offsetWidth,d.$element.addClass("in"),d.enforceFocus();var f=a.Event("shown.bs.modal",{relatedTarget:b});e?d.$dialog.one("bsTransitionEnd",function(){d.$element.trigger("focus").trigger(f)}).emulateTransitionEnd(c.TRANSITION_DURATION):d.$element.trigger("focus").trigger(f)}))},c.prototype.hide=function(b){b&&b.preventDefault(),b=a.Event("hide.bs.modal"),this.$element.trigger(b),this.isShown&&!b.isDefaultPrevented()&&(this.isShown=!1,this.escape(),this.resize(),a(document).off("focusin.bs.modal"),this.$element.removeClass("in").off("click.dismiss.bs.modal").off("mouseup.dismiss.bs.modal"),this.$dialog.off("mousedown.dismiss.bs.m
odal"),a.support.transition&&this.$element.hasClass("fade")?this.$element.one("bsTransitionEnd",a.proxy(this.hideModal,this)).emulateTransitionEnd(c.TRANSITION_DURATION):this.hideModal())},c.prototype.enforceFocus=function(){a(document).off("focusin.bs.modal").on("focusin.bs.modal",a.proxy(function(a){document===a.target||this.$element[0]===a.target||this.$element.has(a.target).length||this.$element.trigger("focus")},this))},c.prototype.escape=function(){this.isShown&&this.options.keyboard?this.$element.on("keydown.dismiss.bs.modal",a.proxy(function(a){27==a.which&&this.hide()},this)):this.isShown||this.$element.off("keydown.dismiss.bs.modal")},c.prototype.resize=function(){this.isShown?a(window).on("resize.bs.modal",a.proxy(this.handleUpdate,this)):a(window).off("resize.bs.modal")},c.prototype.hideModal=function(){var a=this;this.$element.hide(),this.backdrop(function(){a.$body.removeClass("modal-open"),a.resetAdjustments(),a.resetScrollbar(),a.$element.trigger("hidden.bs.modal")})
},c.prototype.removeBackdrop=function(){this.$backdrop&&this.$backdrop.remove(),this.$backdrop=null},c.prototype.backdrop=function(b){var d=this,e=this.$element.hasClass("fade")?"fade":"";if(this.isShown&&this.options.backdrop){var f=a.support.transition&&e;if(this.$backdrop=a(document.createElement("div")).addClass("modal-backdrop "+e).appendTo(this.$body),this.$element.on("click.dismiss.bs.modal",a.proxy(function(a){return this.ignoreBackdropClick?void(this.ignoreBackdropClick=!1):void(a.target===a.currentTarget&&("static"==this.options.backdrop?this.$element[0].focus():this.hide()))},this)),f&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in"),!b)return;f?this.$backdrop.one("bsTransitionEnd",b).emulateTransitionEnd(c.BACKDROP_TRANSITION_DURATION):b()}else if(!this.isShown&&this.$backdrop){this.$backdrop.removeClass("in");var g=function(){d.removeBackdrop(),b&&b()};a.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one("bsTransitionEnd",g).emulateTransiti
onEnd(c.BACKDROP_TRANSITION_DURATION):g()}else b&&b()},c.prototype.handleUpdate=function(){this.adjustDialog()},c.prototype.adjustDialog=function(){var a=this.$element[0].scrollHeight>document.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&a?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!a?this.scrollbarWidth:""})},c.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},c.prototype.checkScrollbar=function(){var a=window.innerWidth;if(!a){var b=document.documentElement.getBoundingClientRect();a=b.right-Math.abs(b.left)}this.bodyIsOverflowing=document.body.clientWidth<a,this.scrollbarWidth=this.measureScrollbar()},c.prototype.setScrollbar=function(){var a=parseInt(this.$body.css("padding-right")||0,10);this.originalBodyPad=document.body.style.paddingRight||"",this.bodyIsOverflowing&&this.$body.css("padding-right",a+this.scrollbarWidth)},c.prototype.resetScrollbar=function(){this.$body.css("padding
-right",this.originalBodyPad)},c.prototype.measureScrollbar=function(){var a=document.createElement("div");a.className="modal-scrollbar-measure",this.$body.append(a);var b=a.offsetWidth-a.clientWidth;return this.$body[0].removeChild(a),b};var d=a.fn.modal;a.fn.modal=b,a.fn.modal.Constructor=c,a.fn.modal.noConflict=function(){return a.fn.modal=d,this},a(document).on("click.bs.modal.data-api",'[data-toggle="modal"]',function(c){var d=a(this),e=d.attr("href"),f=a(d.attr("data-target")||e&&e.replace(/.*(?=#[^\s]+$)/,"")),g=f.data("bs.modal")?"toggle":a.extend({remote:!/#/.test(e)&&e},f.data(),d.data());d.is("a")&&c.preventDefault(),f.one("show.bs.modal",function(a){a.isDefaultPrevented()||f.one("hidden.bs.modal",function(){d.is(":visible")&&d.trigger("focus")})}),b.call(f,g,this)})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.tooltip"),f="object"==typeof b&&b;!e&&/destroy|hide/.test(b)||(e||d.data("bs.tooltip",e=new c(this,f)),"
string"==typeof b&&e[b]())})}var c=function(a,b){this.type=null,this.options=null,this.enabled=null,this.timeout=null,this.hoverState=null,this.$element=null,this.inState=null,this.init("tooltip",a,b)};c.VERSION="3.3.7",c.TRANSITION_DURATION=150,c.DEFAULTS={animation:!0,placement:"top",selector:!1,template:'<div class="tooltip" role="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0}},c.prototype.init=function(b,c,d){if(this.enabled=!0,this.type=b,this.$element=a(c),this.options=this.getOptions(d),this.$viewport=this.options.viewport&&a(a.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+
this.type+" on the window.document object!");for(var e=this.options.trigger.split(" "),f=e.length;f--;){var g=e[f];if("click"==g)this.$element.on("click."+this.type,this.options.selector,a.proxy(this.toggle,this));else if("manual"!=g){var h="hover"==g?"mouseenter":"focusin",i="hover"==g?"mouseleave":"focusout";this.$element.on(h+"."+this.type,this.options.selector,a.proxy(this.enter,this)),this.$element.on(i+"."+this.type,this.options.selector,a.proxy(this.leave,this))}}this.options.selector?this._options=a.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.getOptions=function(b){return b=a.extend({},this.getDefaults(),this.$element.data(),b),b.delay&&"number"==typeof b.delay&&(b.delay={show:b.delay,hide:b.delay}),b},c.prototype.getDelegateOptions=function(){var b={},c=this.getDefaults();return this._options&&a.each(this._options,function(a,d){c[a]!=d&&(b[a]=d)}),b},c.prototype.enter=function(b){v
ar c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusin"==b.type?"focus":"hover"]=!0),c.tip().hasClass("in")||"in"==c.hoverState?void(c.hoverState="in"):(clearTimeout(c.timeout),c.hoverState="in",c.options.delay&&c.options.delay.show?void(c.timeout=setTimeout(function(){"in"==c.hoverState&&c.show()},c.options.delay.show)):c.show())},c.prototype.isInStateTrue=function(){for(var a in this.inState)if(this.inState[a])return!0;return!1},c.prototype.leave=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);if(c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusout"==b.type?"focus":"hover"]=!1),!c.isInStateTrue())return clearTimeout(c.timeout),c.hoverState="out",c.option
s.delay&&c.options.delay.hide?void(c.timeout=setTimeout(function(){"out"==c.hoverState&&c.hide()},c.options.delay.hide)):c.hide()},c.prototype.show=function(){var b=a.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(b);var d=a.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(b.isDefaultPrevented()||!d)return;var e=this,f=this.tip(),g=this.getUID(this.type);this.setContent(),f.attr("id",g),this.$element.attr("aria-describedby",g),this.options.animation&&f.addClass("fade");var h="function"==typeof this.options.placement?this.options.placement.call(this,f[0],this.$element[0]):this.options.placement,i=/\s?auto?\s?/i,j=i.test(h);j&&(h=h.replace(i,"")||"top"),f.detach().css({top:0,left:0,display:"block"}).addClass(h).data("bs."+this.type,this),this.options.container?f.appendTo(this.options.container):f.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type);var k=this.getPosition(),l=f[0].offsetWidth,m=f[0
].offsetHeight;if(j){var n=h,o=this.getPosition(this.$viewport);h="bottom"==h&&k.bottom+m>o.bottom?"top":"top"==h&&k.top-m<o.top?"bottom":"right"==h&&k.right+l>o.width?"left":"left"==h&&k.left-l<o.left?"right":h,f.removeClass(n).addClass(h)}var p=this.getCalculatedOffset(h,k,l,m);this.applyPlacement(p,h);var q=function(){var a=e.hoverState;e.$element.trigger("shown.bs."+e.type),e.hoverState=null,"out"==a&&e.leave(e)};a.support.transition&&this.$tip.hasClass("fade")?f.one("bsTransitionEnd",q).emulateTransitionEnd(c.TRANSITION_DURATION):q()}},c.prototype.applyPlacement=function(b,c){var d=this.tip(),e=d[0].offsetWidth,f=d[0].offsetHeight,g=parseInt(d.css("margin-top"),10),h=parseInt(d.css("margin-left"),10);isNaN(g)&&(g=0),isNaN(h)&&(h=0),b.top+=g,b.left+=h,a.offset.setOffset(d[0],a.extend({using:function(a){d.css({top:Math.round(a.top),left:Math.round(a.left)})}},b),0),d.addClass("in");var i=d[0].offsetWidth,j=d[0].offsetHeight;"top"==c&&j!=f&&(b.top=b.top+f-j);var k=this.getViewport
AdjustedDelta(c,b,i,j);k.left?b.left+=k.left:b.top+=k.top;var l=/top|bottom/.test(c),m=l?2*k.left-e+i:2*k.top-f+j,n=l?"offsetWidth":"offsetHeight";d.offset(b),this.replaceArrow(m,d[0][n],l)},c.prototype.replaceArrow=function(a,b,c){this.arrow().css(c?"left":"top",50*(1-a/b)+"%").css(c?"top":"left","")},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle();a.find(".tooltip-inner")[this.options.html?"html":"text"](b),a.removeClass("fade in top bottom left right")},c.prototype.hide=function(b){function d(){"in"!=e.hoverState&&f.detach(),e.$element&&e.$element.removeAttr("aria-describedby").trigger("hidden.bs."+e.type),b&&b()}var e=this,f=a(this.$tip),g=a.Event("hide.bs."+this.type);if(this.$element.trigger(g),!g.isDefaultPrevented())return f.removeClass("in"),a.support.transition&&f.hasClass("fade")?f.one("bsTransitionEnd",d).emulateTransitionEnd(c.TRANSITION_DURATION):d(),this.hoverState=null,this},c.prototype.fixTitle=function(){var a=this.$element;(a.attr("title")||"s
tring"!=typeof a.attr("data-original-title"))&&a.attr("data-original-title",a.attr("title")||"").attr("title","")},c.prototype.hasContent=function(){return this.getTitle()},c.prototype.getPosition=function(b){b=b||this.$element;var c=b[0],d="BODY"==c.tagName,e=c.getBoundingClientRect();null==e.width&&(e=a.extend({},e,{width:e.right-e.left,height:e.bottom-e.top}));var f=window.SVGElement&&c instanceof window.SVGElement,g=d?{top:0,left:0}:f?null:b.offset(),h={scroll:d?document.documentElement.scrollTop||document.body.scrollTop:b.scrollTop()},i=d?{width:a(window).width(),height:a(window).height()}:null;return a.extend({},e,h,i,g)},c.prototype.getCalculatedOffset=function(a,b,c,d){return"bottom"==a?{top:b.top+b.height,left:b.left+b.width/2-c/2}:"top"==a?{top:b.top-d,left:b.left+b.width/2-c/2}:"left"==a?{top:b.top+b.height/2-d/2,left:b.left-c}:{top:b.top+b.height/2-d/2,left:b.left+b.width}},c.prototype.getViewportAdjustedDelta=function(a,b,c,d){var e={top:0,left:0};if(!this.$viewport)ret
urn e;var f=this.options.viewport&&this.options.viewport.padding||0,g=this.getPosition(this.$viewport);if(/right|left/.test(a)){var h=b.top-f-g.scroll,i=b.top+f-g.scroll+d;h<g.top?e.top=g.top-h:i>g.top+g.height&&(e.top=g.top+g.height-i)}else{var j=b.left-f,k=b.left+f+c;j<g.left?e.left=g.left-j:k>g.right&&(e.left=g.left+g.width-k)}return e},c.prototype.getTitle=function(){var a,b=this.$element,c=this.options;return a=b.attr("data-original-title")||("function"==typeof c.title?c.title.call(b[0]):c.title)},c.prototype.getUID=function(a){do a+=~~(1e6*Math.random());while(document.getElementById(a));return a},c.prototype.tip=function(){if(!this.$tip&&(this.$tip=a(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},c.prototype.enable=function(){this.enabled=!0},c.prototype.disable=function(){th
is.enabled=!1},c.prototype.toggleEnabled=function(){this.enabled=!this.enabled},c.prototype.toggle=function(b){var c=this;b&&(c=a(b.currentTarget).data("bs."+this.type),c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c))),b?(c.inState.click=!c.inState.click,c.isInStateTrue()?c.enter(c):c.leave(c)):c.tip().hasClass("in")?c.leave(c):c.enter(c)},c.prototype.destroy=function(){var a=this;clearTimeout(this.timeout),this.hide(function(){a.$element.off("."+a.type).removeData("bs."+a.type),a.$tip&&a.$tip.detach(),a.$tip=null,a.$arrow=null,a.$viewport=null,a.$element=null})};var d=a.fn.tooltip;a.fn.tooltip=b,a.fn.tooltip.Constructor=c,a.fn.tooltip.noConflict=function(){return a.fn.tooltip=d,this}}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.popover"),f="object"==typeof b&&b;!e&&/destroy|hide/.test(b)||(e||d.data("bs.popover",e=new c(this,f)),"string"==typeof b&&e[b]())})}v
ar c=function(a,b){this.init("popover",a,b)};if(!a.fn.tooltip)throw new Error("Popover requires tooltip.js");c.VERSION="3.3.7",c.DEFAULTS=a.extend({},a.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:'<div class="popover" role="tooltip"><div class="arrow"></div><h3 class="popover-title"></h3><div class="popover-content"></div></div>'}),c.prototype=a.extend({},a.fn.tooltip.Constructor.prototype),c.prototype.constructor=c,c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle(),c=this.getContent();a.find(".popover-title")[this.options.html?"html":"text"](b),a.find(".popover-content").children().detach().end()[this.options.html?"string"==typeof c?"html":"append":"text"](c),a.removeClass("fade top bottom left right in"),a.find(".popover-title").html()||a.find(".popover-title").hide()},c.prototype.hasContent=function(){return this.getTitle()||this.getContent()},c.prototype.getContent=
function(){var a=this.$element,b=this.options;return a.attr("data-content")||("function"==typeof b.content?b.content.call(a[0]):b.content)},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var d=a.fn.popover;a.fn.popover=b,a.fn.popover.Constructor=c,a.fn.popover.noConflict=function(){return a.fn.popover=d,this}}(jQuery),+function(a){"use strict";function b(c,d){this.$body=a(document.body),this.$scrollElement=a(a(c).is(document.body)?window:c),this.options=a.extend({},b.DEFAULTS,d),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",a.proxy(this.process,this)),this.refresh(),this.process()}function c(c){return this.each(function(){var d=a(this),e=d.data("bs.scrollspy"),f="object"==typeof c&&c;e||d.data("bs.scrollspy",e=new b(this,f)),"string"==typeof c&&e[c]()})}b.VERSION="3.3.7",b.DEFAULTS={offset:10},b.prototype.getScrollH
eight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},b.prototype.refresh=function(){var b=this,c="offset",d=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),a.isWindow(this.$scrollElement[0])||(c="position",d=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var b=a(this),e=b.data("target")||b.attr("href"),f=/^#./.test(e)&&a(e);return f&&f.length&&f.is(":visible")&&[[f[c]().top+d,e]]||null}).sort(function(a,b){return a[0]-b[0]}).each(function(){b.offsets.push(this[0]),b.targets.push(this[1])})},b.prototype.process=function(){var a,b=this.$scrollElement.scrollTop()+this.options.offset,c=this.getScrollHeight(),d=this.options.offset+c-this.$scrollElement.height(),e=this.offsets,f=this.targets,g=this.activeTarget;if(this.scrollHeight!=c&&this.refresh(),b>=d)return g!=(a=f[f.length-1])&&this.activate(a);if(g&&b<e[0])return this.activeTarget=null,this.
clear();for(a=e.length;a--;)g!=f[a]&&b>=e[a]&&(void 0===e[a+1]||b<e[a+1])&&this.activate(f[a])},b.prototype.activate=function(b){
+this.activeTarget=b,this.clear();var c=this.selector+'[data-target="'+b+'"],'+this.selector+'[href="'+b+'"]',d=a(c).parents("li").addClass("active");d.parent(".dropdown-menu").length&&(d=d.closest("li.dropdown").addClass("active")),d.trigger("activate.bs.scrollspy")},b.prototype.clear=function(){a(this.selector).parentsUntil(this.options.target,".active").removeClass("active")};var d=a.fn.scrollspy;a.fn.scrollspy=c,a.fn.scrollspy.Constructor=b,a.fn.scrollspy.noConflict=function(){return a.fn.scrollspy=d,this},a(window).on("load.bs.scrollspy.data-api",function(){a('[data-spy="scroll"]').each(function(){var b=a(this);c.call(b,b.data())})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.tab");e||d.data("bs.tab",e=new c(this)),"string"==typeof b&&e[b]()})}var c=function(b){this.element=a(b)};c.VERSION="3.3.7",c.TRANSITION_DURATION=150,c.prototype.show=function(){var b=this.element,c=b.closest("ul:not(.dropdown-menu)"),d=b.data("ta
rget");if(d||(d=b.attr("href"),d=d&&d.replace(/.*(?=#[^\s]*$)/,"")),!b.parent("li").hasClass("active")){var e=c.find(".active:last a"),f=a.Event("hide.bs.tab",{relatedTarget:b[0]}),g=a.Event("show.bs.tab",{relatedTarget:e[0]});if(e.trigger(f),b.trigger(g),!g.isDefaultPrevented()&&!f.isDefaultPrevented()){var h=a(d);this.activate(b.closest("li"),c),this.activate(h,h.parent(),function(){e.trigger({type:"hidden.bs.tab",relatedTarget:b[0]}),b.trigger({type:"shown.bs.tab",relatedTarget:e[0]})})}}},c.prototype.activate=function(b,d,e){function f(){g.removeClass("active").find("> .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),b.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),h?(b[0].offsetWidth,b.addClass("in")):b.removeClass("fade"),b.parent(".dropdown-menu").length&&b.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),e&&e()}var g=d.find("> .active"),h=e&
&a.support.transition&&(g.length&&g.hasClass("fade")||!!d.find("> .fade").length);g.length&&h?g.one("bsTransitionEnd",f).emulateTransitionEnd(c.TRANSITION_DURATION):f(),g.removeClass("in")};var d=a.fn.tab;a.fn.tab=b,a.fn.tab.Constructor=c,a.fn.tab.noConflict=function(){return a.fn.tab=d,this};var e=function(c){c.preventDefault(),b.call(a(this),"show")};a(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',e).on("click.bs.tab.data-api",'[data-toggle="pill"]',e)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.affix"),f="object"==typeof b&&b;e||d.data("bs.affix",e=new c(this,f)),"string"==typeof b&&e[b]()})}var c=function(b,d){this.options=a.extend({},c.DEFAULTS,d),this.$target=a(this.options.target).on("scroll.bs.affix.data-api",a.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",a.proxy(this.checkPositionWithEventLoop,this)),this.$element=a(b),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkP
osition()};c.VERSION="3.3.7",c.RESET="affix affix-top affix-bottom",c.DEFAULTS={offset:0,target:window},c.prototype.getState=function(a,b,c,d){var e=this.$target.scrollTop(),f=this.$element.offset(),g=this.$target.height();if(null!=c&&"top"==this.affixed)return e<c&&"top";if("bottom"==this.affixed)return null!=c?!(e+this.unpin<=f.top)&&"bottom":!(e+g<=a-d)&&"bottom";var h=null==this.affixed,i=h?e:f.top,j=h?g:b;return null!=c&&e<=c?"top":null!=d&&i+j>=a-d&&"bottom"},c.prototype.getPinnedOffset=function(){if(this.pinnedOffset)return this.pinnedOffset;this.$element.removeClass(c.RESET).addClass("affix");var a=this.$target.scrollTop(),b=this.$element.offset();return this.pinnedOffset=b.top-a},c.prototype.checkPositionWithEventLoop=function(){setTimeout(a.proxy(this.checkPosition,this),1)},c.prototype.checkPosition=function(){if(this.$element.is(":visible")){var b=this.$element.height(),d=this.options.offset,e=d.top,f=d.bottom,g=Math.max(a(document).height(),a(document.body).height());"o
bject"!=typeof d&&(f=e=d),"function"==typeof e&&(e=d.top(this.$element)),"function"==typeof f&&(f=d.bottom(this.$element));var h=this.getState(g,b,e,f);if(this.affixed!=h){null!=this.unpin&&this.$element.css("top","");var i="affix"+(h?"-"+h:""),j=a.Event(i+".bs.affix");if(this.$element.trigger(j),j.isDefaultPrevented())return;this.affixed=h,this.unpin="bottom"==h?this.getPinnedOffset():null,this.$element.removeClass(c.RESET).addClass(i).trigger(i.replace("affix","affixed")+".bs.affix")}"bottom"==h&&this.$element.offset({top:g-b-f})}};var d=a.fn.affix;a.fn.affix=b,a.fn.affix.Constructor=c,a.fn.affix.noConflict=function(){return a.fn.affix=d,this},a(window).on("load",function(){a('[data-spy="affix"]').each(function(){var c=a(this),d=c.data();d.offset=d.offset||{},null!=d.offsetBottom&&(d.offset.bottom=d.offsetBottom),null!=d.offsetTop&&(d.offset.top=d.offsetTop),b.call(c,d)})})}(jQuery);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/dataTables.bootstrap.min.css
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/dataTables.bootstrap.min.css b/server/src/main/resources/org/apache/livy/server/ui/static/dataTables.bootstrap.min.css
new file mode 100644
index 0000000..66a70ab
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/dataTables.bootstrap.min.css
@@ -0,0 +1 @@
+table.dataTable{clear:both;margin-top:6px !important;margin-bottom:6px !important;max-width:none !important;border-collapse:separate !important}table.dataTable td,table.dataTable th{-webkit-box-sizing:content-box;box-sizing:content-box}table.dataTable td.dataTables_empty,table.dataTable th.dataTables_empty{text-align:center}table.dataTable.nowrap th,table.dataTable.nowrap td{white-space:nowrap}div.dataTables_wrapper div.dataTables_length label{font-weight:normal;text-align:left;white-space:nowrap}div.dataTables_wrapper div.dataTables_length select{width:75px;display:inline-block}div.dataTables_wrapper div.dataTables_filter{text-align:right}div.dataTables_wrapper div.dataTables_filter label{font-weight:normal;white-space:nowrap;text-align:left}div.dataTables_wrapper div.dataTables_filter input{margin-left:0.5em;display:inline-block;width:auto}div.dataTables_wrapper div.dataTables_info{padding-top:8px;white-space:nowrap}div.dataTables_wrapper div.dataTables_paginate{margin:0;white-spa
ce:nowrap;text-align:right}div.dataTables_wrapper div.dataTables_paginate ul.pagination{margin:2px 0;white-space:nowrap}div.dataTables_wrapper div.dataTables_processing{position:absolute;top:50%;left:50%;width:200px;margin-left:-100px;margin-top:-26px;text-align:center;padding:1em 0}table.dataTable thead>tr>th.sorting_asc,table.dataTable thead>tr>th.sorting_desc,table.dataTable thead>tr>th.sorting,table.dataTable thead>tr>td.sorting_asc,table.dataTable thead>tr>td.sorting_desc,table.dataTable thead>tr>td.sorting{padding-right:30px}table.dataTable thead>tr>th:active,table.dataTable thead>tr>td:active{outline:none}table.dataTable thead .sorting,table.dataTable thead .sorting_asc,table.dataTable thead .sorting_desc,table.dataTable thead .sorting_asc_disabled,table.dataTable thead .sorting_desc_disabled{cursor:pointer;position:relative}table.dataTable thead .sorting:after,table.dataTable thead .sorting_asc:after,table.dataTable thead .sorting_desc:after,table.dataTable thead .sorting_as
c_disabled:after,table.dataTable thead .sorting_desc_disabled:after{position:absolute;bottom:8px;right:8px;display:block;font-family:'Glyphicons Halflings';opacity:0.5}table.dataTable thead .sorting:after{opacity:0.2;content:"\e150"}table.dataTable thead .sorting_asc:after{content:"\e155"}table.dataTable thead .sorting_desc:after{content:"\e156"}table.dataTable thead .sorting_asc_disabled:after,table.dataTable thead .sorting_desc_disabled:after{color:#eee}div.dataTables_scrollHead table.dataTable{margin-bottom:0 !important}div.dataTables_scrollBody>table{border-top:none;margin-top:0 !important;margin-bottom:0 !important}div.dataTables_scrollBody>table>thead .sorting:after,div.dataTables_scrollBody>table>thead .sorting_asc:after,div.dataTables_scrollBody>table>thead .sorting_desc:after{display:none}div.dataTables_scrollBody>table>tbody>tr:first-child>th,div.dataTables_scrollBody>table>tbody>tr:first-child>td{border-top:none}div.dataTables_scrollFoot>table{margin-top:0 !important;bord
er-top:none}@media screen and (max-width: 767px){div.dataTables_wrapper div.dataTables_length,div.dataTables_wrapper div.dataTables_filter,div.dataTables_wrapper div.dataTables_info,div.dataTables_wrapper div.dataTables_paginate{text-align:center}}table.dataTable.table-condensed>thead>tr>th{padding-right:20px}table.dataTable.table-condensed .sorting:after,table.dataTable.table-condensed .sorting_asc:after,table.dataTable.table-condensed .sorting_desc:after{top:6px;right:6px}table.table-bordered.dataTable th,table.table-bordered.dataTable td{border-left-width:0}table.table-bordered.dataTable th:last-child,table.table-bordered.dataTable th:last-child,table.table-bordered.dataTable td:last-child,table.table-bordered.dataTable td:last-child{border-right-width:0}table.table-bordered.dataTable tbody th,table.table-bordered.dataTable tbody td{border-bottom-width:0}div.dataTables_scrollHead table.table-bordered{border-bottom-width:0}div.table-responsive>div.dataTables_wrapper>div.row{margin
:0}div.table-responsive>div.dataTables_wrapper>div.row>div[class^="col-"]:first-child{padding-left:0}div.table-responsive>div.dataTables_wrapper>div.row>div[class^="col-"]:last-child{padding-right:0}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/dataTables.bootstrap.min.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/dataTables.bootstrap.min.js b/server/src/main/resources/org/apache/livy/server/ui/static/dataTables.bootstrap.min.js
new file mode 100644
index 0000000..98661c6
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/dataTables.bootstrap.min.js
@@ -0,0 +1,8 @@
+/*!
+ DataTables Bootstrap 3 integration
+ ©2011-2015 SpryMedia Ltd - datatables.net/license
+*/
+(function(b){"function"===typeof define&&define.amd?define(["jquery","datatables.net"],function(a){return b(a,window,document)}):"object"===typeof exports?module.exports=function(a,d){a||(a=window);if(!d||!d.fn.dataTable)d=require("datatables.net")(a,d).$;return b(d,a,a.document)}:b(jQuery,window,document)})(function(b,a,d,m){var f=b.fn.dataTable;b.extend(!0,f.defaults,{dom:"<'row'<'col-sm-6'l><'col-sm-6'f>><'row'<'col-sm-12'tr>><'row'<'col-sm-5'i><'col-sm-7'p>>",renderer:"bootstrap"});b.extend(f.ext.classes,
+{sWrapper:"dataTables_wrapper form-inline dt-bootstrap",sFilterInput:"form-control input-sm",sLengthSelect:"form-control input-sm",sProcessing:"dataTables_processing panel panel-default"});f.ext.renderer.pageButton.bootstrap=function(a,h,r,s,j,n){var o=new f.Api(a),t=a.oClasses,k=a.oLanguage.oPaginate,u=a.oLanguage.oAria.paginate||{},e,g,p=0,q=function(d,f){var l,h,i,c,m=function(a){a.preventDefault();!b(a.currentTarget).hasClass("disabled")&&o.page()!=a.data.action&&o.page(a.data.action).draw("page")};
+l=0;for(h=f.length;l<h;l++)if(c=f[l],b.isArray(c))q(d,c);else{g=e="";switch(c){case "ellipsis":e="…";g="disabled";break;case "first":e=k.sFirst;g=c+(0<j?"":" disabled");break;case "previous":e=k.sPrevious;g=c+(0<j?"":" disabled");break;case "next":e=k.sNext;g=c+(j<n-1?"":" disabled");break;case "last":e=k.sLast;g=c+(j<n-1?"":" disabled");break;default:e=c+1,g=j===c?"active":""}e&&(i=b("<li>",{"class":t.sPageButton+" "+g,id:0===r&&"string"===typeof c?a.sTableId+"_"+c:null}).append(b("<a>",{href:"#",
+"aria-controls":a.sTableId,"aria-label":u[c],"data-dt-idx":p,tabindex:a.iTabIndex}).html(e)).appendTo(d),a.oApi._fnBindAction(i,{action:c},m),p++)}},i;try{i=b(h).find(d.activeElement).data("dt-idx")}catch(v){}q(b(h).empty().html('<ul class="pagination"/>').children("ul"),s);i!==m&&b(h).find("[data-dt-idx="+i+"]").focus()};return f});
[32/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/com/cloudera/livy/client/common/AbstractJobHandle.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/com/cloudera/livy/client/common/AbstractJobHandle.java b/client-common/src/main/java/com/cloudera/livy/client/common/AbstractJobHandle.java
deleted file mode 100644
index cb1b4e8..0000000
--- a/client-common/src/main/java/com/cloudera/livy/client/common/AbstractJobHandle.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-
-import com.cloudera.livy.JobHandle;
-import com.cloudera.livy.annotations.Private;
-
-@Private
-public abstract class AbstractJobHandle<T> implements JobHandle<T> {
-
- protected final List<Listener<T>> listeners;
- protected volatile State state;
-
- protected AbstractJobHandle() {
- this.listeners = new LinkedList<>();
- this.state = State.SENT;
- }
-
- @Override
- public State getState() {
- return state;
- }
-
- @Override
- public void addListener(Listener<T> l) {
- synchronized (listeners) {
- listeners.add(l);
- fireStateChange(state, l);
- }
- }
-
- /**
- * Changes the state of this job handle, making sure that illegal state transitions are ignored.
- * Fires events appropriately.
- *
- * As a rule, state transitions can only occur if the current state is "higher" than the current
- * state (i.e., has a higher ordinal number) and is not a "final" state. "Final" states are
- * CANCELLED, FAILED and SUCCEEDED, defined here in the code as having an ordinal number higher
- * than the CANCELLED enum constant.
- */
- public boolean changeState(State newState) {
- synchronized (listeners) {
- if (newState.ordinal() > state.ordinal() && state.ordinal() < State.CANCELLED.ordinal()) {
- state = newState;
- for (Listener<T> l : listeners) {
- fireStateChange(newState, l);
- }
- return true;
- }
- return false;
- }
- }
-
- protected abstract T result();
- protected abstract Throwable error();
-
- private void fireStateChange(State s, Listener<T> l) {
- switch (s) {
- case SENT:
- break;
- case QUEUED:
- l.onJobQueued(this);
- break;
- case STARTED:
- l.onJobStarted(this);
- break;
- case CANCELLED:
- l.onJobCancelled(this);
- break;
- case FAILED:
- l.onJobFailed(this, error());
- break;
- case SUCCEEDED:
- try {
- l.onJobSucceeded(this, result());
- } catch (Exception e) {
- // Shouldn't really happen.
- throw new IllegalStateException(e);
- }
- break;
- default:
- throw new IllegalStateException();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/com/cloudera/livy/client/common/BufferUtils.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/com/cloudera/livy/client/common/BufferUtils.java b/client-common/src/main/java/com/cloudera/livy/client/common/BufferUtils.java
deleted file mode 100644
index 3084ae5..0000000
--- a/client-common/src/main/java/com/cloudera/livy/client/common/BufferUtils.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.nio.ByteBuffer;
-
-import com.cloudera.livy.annotations.Private;
-
-/**
- * Utility methods for dealing with byte buffers and byte arrays.
- */
-@Private
-public class BufferUtils {
-
- public static byte[] toByteArray(ByteBuffer buf) {
- byte[] bytes;
- if (buf.hasArray() && buf.arrayOffset() == 0 &&
- buf.remaining() == buf.array().length) {
- bytes = buf.array();
- } else {
- bytes = new byte[buf.remaining()];
- buf.get(bytes);
- }
- return bytes;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/com/cloudera/livy/client/common/ClientConf.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/com/cloudera/livy/client/common/ClientConf.java b/client-common/src/main/java/com/cloudera/livy/client/common/ClientConf.java
deleted file mode 100644
index 035c75e..0000000
--- a/client-common/src/main/java/com/cloudera/livy/client/common/ClientConf.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.annotations.Private;
-
-/**
- * Base class with common functionality for type-safe configuration objects.
- */
-@Private
-public abstract class ClientConf<T extends ClientConf>
- implements Iterable<Map.Entry<String, String>> {
-
- protected Logger LOG = LoggerFactory.getLogger(getClass());
-
- public static interface ConfEntry {
-
- /** The key in the configuration file. */
- String key();
-
- /**
- * The default value, which also defines the type of the config. Supported types:
- * Boolean, Integer, Long, String. <code>null</code> maps to String.
- */
- Object dflt();
-
- }
-
- private static final Map<String, TimeUnit> TIME_SUFFIXES;
-
- public static final boolean TEST_MODE = Boolean.parseBoolean(System.getenv("LIVY_TEST"));
-
- static {
- TIME_SUFFIXES = new HashMap<>();
- TIME_SUFFIXES.put("us", TimeUnit.MICROSECONDS);
- TIME_SUFFIXES.put("ms", TimeUnit.MILLISECONDS);
- TIME_SUFFIXES.put("s", TimeUnit.SECONDS);
- TIME_SUFFIXES.put("m", TimeUnit.MINUTES);
- TIME_SUFFIXES.put("min", TimeUnit.MINUTES);
- TIME_SUFFIXES.put("h", TimeUnit.HOURS);
- TIME_SUFFIXES.put("d", TimeUnit.DAYS);
- }
-
- protected final ConcurrentMap<String, String> config;
-
- protected ClientConf(Properties config) {
- this.config = new ConcurrentHashMap<>();
- if (config != null) {
- for (String key : config.stringPropertyNames()) {
- logDeprecationWarning(key);
- this.config.put(key, config.getProperty(key));
- }
- }
- }
-
- public String get(String key) {
- String val = config.get(key);
- if (val != null) {
- return val;
- }
- DeprecatedConf depConf = getConfigsWithAlternatives().get(key);
- if (depConf != null) {
- return config.get(depConf.key());
- } else {
- return val;
- }
- }
-
- @SuppressWarnings("unchecked")
- public T set(String key, String value) {
- logDeprecationWarning(key);
- config.put(key, value);
- return (T) this;
- }
-
- @SuppressWarnings("unchecked")
- public T setIfMissing(String key, String value) {
- if (config.putIfAbsent(key, value) == null) {
- logDeprecationWarning(key);
- }
- return (T) this;
- }
-
- @SuppressWarnings("unchecked")
- public T setAll(ClientConf<?> other) {
- for (Map.Entry<String, String> e : other) {
- set(e.getKey(), e.getValue());
- }
- return (T) this;
- }
-
- public String get(ConfEntry e) {
- Object value = get(e, String.class);
- return (String) (value != null ? value : e.dflt());
- }
-
- public boolean getBoolean(ConfEntry e) {
- String val = get(e, Boolean.class);
- if (val != null) {
- return Boolean.parseBoolean(val);
- } else {
- return (Boolean) e.dflt();
- }
- }
-
- public int getInt(ConfEntry e) {
- String val = get(e, Integer.class);
- if (val != null) {
- return Integer.parseInt(val);
- } else {
- return (Integer) e.dflt();
- }
- }
-
- public long getLong(ConfEntry e) {
- String val = get(e, Long.class);
- if (val != null) {
- return Long.parseLong(val);
- } else {
- return (Long) e.dflt();
- }
- }
-
- public long getTimeAsMs(ConfEntry e) {
- String time = get(e, String.class);
- if (time == null) {
- check(e.dflt() != null,
- "ConfEntry %s doesn't have a default value, cannot convert to time value.", e.key());
- time = (String) e.dflt();
- }
-
- Matcher m = Pattern.compile("(-?[0-9]+)([a-z]+)?").matcher(time.toLowerCase());
- if (!m.matches()) {
- throw new IllegalArgumentException("Invalid time string: " + time);
- }
-
- long val = Long.parseLong(m.group(1));
- String suffix = m.group(2);
-
- if (suffix != null && !TIME_SUFFIXES.containsKey(suffix)) {
- throw new IllegalArgumentException("Invalid suffix: \"" + suffix + "\"");
- }
-
- return TimeUnit.MILLISECONDS.convert(val,
- suffix != null ? TIME_SUFFIXES.get(suffix) : TimeUnit.MILLISECONDS);
- }
-
- @SuppressWarnings("unchecked")
- public T set(ConfEntry e, Object value) {
- check(typesMatch(value, e.dflt()), "Value doesn't match configuration entry type for %s.",
- e.key());
- if (value == null) {
- config.remove(e.key());
- } else {
- logDeprecationWarning(e.key());
- config.put(e.key(), value.toString());
- }
- return (T) this;
- }
-
- @Override
- public Iterator<Map.Entry<String, String>> iterator() {
- return config.entrySet().iterator();
- }
-
- private String get(ConfEntry e, Class<?> requestedType) {
- check(getType(e.dflt()).equals(requestedType), "Invalid type conversion requested for %s.",
- e.key());
- return this.get(e.key());
- }
-
- private boolean typesMatch(Object test, Object expected) {
- return test == null || getType(test).equals(getType(expected));
- }
-
- private Class<?> getType(Object o) {
- return (o != null) ? o.getClass() : String.class;
- }
-
- private void check(boolean test, String message, Object... args) {
- if (!test) {
- throw new IllegalArgumentException(String.format(message, args));
- }
- }
-
- /** Logs a warning message if the given config key is deprecated. */
- private void logDeprecationWarning(String key) {
- DeprecatedConf altConfs = getConfigsWithAlternatives().get(key);
- if (altConfs != null) {
- LOG.warn("The configuration key " + altConfs.key() + " has been deprecated as of Livy "
- + altConfs.version() + " and may be removed in the future. Please use the new key "
- + key + " instead.");
- return;
- }
-
- DeprecatedConf depConfs = getDeprecatedConfigs().get(key);
- if (depConfs != null) {
- LOG.warn("The configuration key " + depConfs.key() + " has been deprecated as of Livy "
- + depConfs.version() + " and may be removed in the future. "
- + depConfs.deprecationMessage());
- }
- }
-
- /** Maps valid key to DeprecatedConf with the deprecated key. */
- protected abstract Map<String, DeprecatedConf> getConfigsWithAlternatives();
-
- /** Maps deprecated key to DeprecatedConf with the same key. */
- protected abstract Map<String, DeprecatedConf> getDeprecatedConfigs();
-
- public static interface DeprecatedConf {
-
- /** The key in the configuration file. */
- String key();
-
- /** The Livy version in which the key was deprecated. */
- String version();
-
- /** Message to include in the deprecation warning for configs without alternatives */
- String deprecationMessage();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/com/cloudera/livy/client/common/HttpMessages.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/com/cloudera/livy/client/common/HttpMessages.java b/client-common/src/main/java/com/cloudera/livy/client/common/HttpMessages.java
deleted file mode 100644
index 28a86bf..0000000
--- a/client-common/src/main/java/com/cloudera/livy/client/common/HttpMessages.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.util.List;
-import java.util.Map;
-
-import com.cloudera.livy.JobHandle.State;
-import com.cloudera.livy.annotations.Private;
-
-/**
- * There are the Java representations of the JSON messages used by the client protocol.
- *
- * Note that Jackson requires an empty constructor (or annotations) to be able to instantiate
- * types, so the extra noise is necessary here.
- */
-@Private
-public class HttpMessages {
-
- public static interface ClientMessage {
-
- }
-
- public static class CreateClientRequest implements ClientMessage {
-
- public final Map<String, String> conf;
-
- public CreateClientRequest(Map<String, String> conf) {
- this.conf = conf;
- }
-
- private CreateClientRequest() {
- this(null);
- }
-
- }
-
- public static class SessionInfo implements ClientMessage {
-
- public final int id;
- public final String appId;
- public final String owner;
- public final String proxyUser;
- public final String state;
- public final String kind;
- public final Map<String, String> appInfo;
- public final List<String> log;
-
- public SessionInfo(int id, String appId, String owner, String proxyUser, String state,
- String kind, Map<String, String> appInfo, List<String> log) {
- this.id = id;
- this.appId = appId;
- this.owner = owner;
- this.proxyUser = proxyUser;
- this.state = state;
- this.kind = kind;
- this.appInfo = appInfo;
- this.log = log;
- }
-
- private SessionInfo() {
- this(-1, null, null, null, null, null, null, null);
- }
-
- }
-
- public static class SerializedJob implements ClientMessage {
-
- public final byte[] job;
-
- public SerializedJob(byte[] job) {
- this.job = job;
- }
-
- private SerializedJob() {
- this(null);
- }
-
- }
-
- public static class AddResource implements ClientMessage {
-
- public final String uri;
-
- public AddResource(String uri) {
- this.uri = uri;
- }
-
- private AddResource() {
- this(null);
- }
-
- }
-
- public static class JobStatus implements ClientMessage {
-
- public final long id;
- public final State state;
- public final byte[] result;
- public final String error;
-
- public JobStatus(long id, State state, byte[] result, String error) {
- this.id = id;
- this.state = state;
- this.error = error;
-
- // json4s, at least, seems confused about whether a "null" in the JSON payload should
- // become a null array or a byte array with length 0. Since there shouldn't be any
- // valid serialized object in a byte array of size 0, translate that to null.
- this.result = (result != null && result.length > 0) ? result : null;
-
- if (this.result != null && state != State.SUCCEEDED) {
- throw new IllegalArgumentException("Result cannot be set unless job succeeded.");
- }
- // The check for "result" is not completely correct, but is here to make the unit tests work.
- if (this.result == null && error != null && state != State.FAILED) {
- throw new IllegalArgumentException("Error cannot be set unless job failed.");
- }
- }
-
- private JobStatus() {
- this(-1, null, null, null);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/com/cloudera/livy/client/common/Serializer.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/com/cloudera/livy/client/common/Serializer.java b/client-common/src/main/java/com/cloudera/livy/client/common/Serializer.java
deleted file mode 100644
index 29d82d4..0000000
--- a/client-common/src/main/java/com/cloudera/livy/client/common/Serializer.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.io.ByteArrayOutputStream;
-import java.nio.ByteBuffer;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import com.esotericsoftware.shaded.org.objenesis.strategy.StdInstantiatorStrategy;
-
-import com.cloudera.livy.annotations.Private;
-
-/**
- * Utility class to serialize user data using Kryo.
- */
-@Private
-public class Serializer {
-
- // Kryo docs say 0-8 are taken. Strange things happen if you don't set an ID when registering
- // classes.
- private static final int REG_ID_BASE = 16;
-
- private final ThreadLocal<Kryo> kryos;
-
- public Serializer(final Class<?>... klasses) {
- this.kryos = new ThreadLocal<Kryo>() {
- @Override
- protected Kryo initialValue() {
- Kryo kryo = new Kryo();
- int count = 0;
- for (Class<?> klass : klasses) {
- kryo.register(klass, REG_ID_BASE + count);
- count++;
- }
- kryo.setInstantiatorStrategy(new StdInstantiatorStrategy());
- kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
- return kryo;
- }
- };
- }
-
- public Object deserialize(ByteBuffer data) {
- byte[] b = new byte[data.remaining()];
- data.get(b);
- Input kryoIn = new Input(b);
- return kryos.get().readClassAndObject(kryoIn);
- }
-
- public ByteBuffer serialize(Object data) {
- ByteBufferOutputStream out = new ByteBufferOutputStream();
- Output kryoOut = new Output(out);
- kryos.get().writeClassAndObject(kryoOut, data);
- kryoOut.flush();
- return out.getBuffer();
- }
-
- private static class ByteBufferOutputStream extends ByteArrayOutputStream {
-
- public ByteBuffer getBuffer() {
- ByteBuffer result = ByteBuffer.wrap(buf, 0, count);
- buf = null;
- reset();
- return result;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/com/cloudera/livy/client/common/TestUtils.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/com/cloudera/livy/client/common/TestUtils.java b/client-common/src/main/java/com/cloudera/livy/client/common/TestUtils.java
deleted file mode 100644
index 18bb13a..0000000
--- a/client-common/src/main/java/com/cloudera/livy/client/common/TestUtils.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.StandardOpenOption;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.cloudera.livy.annotations.Private;
-
-/**
- * Utility methods used by Livy tests.
- */
-@Private
-public class TestUtils {
-
- /**
- * Returns JVM arguments that enable jacoco on a process to be run. The returned arguments
- * create a new, unique output file in the same directory referenced by the "jacoco.args"
- * system property.
- *
- * @return JVM arguments, or null.
- */
- public static String getJacocoArgs() {
- String jacocoArgs = System.getProperty("jacoco.args");
- if (jacocoArgs == null) {
- return null;
- }
-
- Pattern p = Pattern.compile("(.+?destfile=)(.+?)(,.+)?");
- Matcher m = p.matcher(jacocoArgs);
- if (!m.matches()) {
- return null;
- }
-
- String fileName = new File(m.group(2)).getName();
- File outputDir = new File(m.group(2)).getParentFile();
-
- File newFile;
- while (true) {
- int newId = outputDir.list().length;
- newFile = new File(outputDir, "jacoco-" + newId + ".exec");
- try {
- Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW).close();
- break;
- } catch (IOException ioe) {
- // Try again.
- }
- }
-
- StringBuilder newArgs = new StringBuilder();
- newArgs.append(m.group(1));
- newArgs.append(newFile.getAbsolutePath());
- if (m.group(3) != null) {
- newArgs.append(m.group(3));
- }
-
- return newArgs.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/org/apache/livy/client/common/AbstractJobHandle.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/org/apache/livy/client/common/AbstractJobHandle.java b/client-common/src/main/java/org/apache/livy/client/common/AbstractJobHandle.java
new file mode 100644
index 0000000..8d3c307
--- /dev/null
+++ b/client-common/src/main/java/org/apache/livy/client/common/AbstractJobHandle.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.livy.JobHandle;
+import org.apache.livy.annotations.Private;
+
+@Private
+public abstract class AbstractJobHandle<T> implements JobHandle<T> {
+
+ protected final List<Listener<T>> listeners;
+ protected volatile State state;
+
+ protected AbstractJobHandle() {
+ this.listeners = new LinkedList<>();
+ this.state = State.SENT;
+ }
+
+ @Override
+ public State getState() {
+ return state;
+ }
+
+ @Override
+ public void addListener(Listener<T> l) {
+ synchronized (listeners) {
+ listeners.add(l);
+ fireStateChange(state, l);
+ }
+ }
+
+ /**
+ * Changes the state of this job handle, making sure that illegal state transitions are ignored.
+ * Fires events appropriately.
+ *
+ * As a rule, state transitions can only occur if the current state is "higher" than the current
+ * state (i.e., has a higher ordinal number) and is not a "final" state. "Final" states are
+ * CANCELLED, FAILED and SUCCEEDED, defined here in the code as having an ordinal number higher
+ * than the CANCELLED enum constant.
+ */
+ public boolean changeState(State newState) {
+ synchronized (listeners) {
+ if (newState.ordinal() > state.ordinal() && state.ordinal() < State.CANCELLED.ordinal()) {
+ state = newState;
+ for (Listener<T> l : listeners) {
+ fireStateChange(newState, l);
+ }
+ return true;
+ }
+ return false;
+ }
+ }
+
+ protected abstract T result();
+ protected abstract Throwable error();
+
+ private void fireStateChange(State s, Listener<T> l) {
+ switch (s) {
+ case SENT:
+ break;
+ case QUEUED:
+ l.onJobQueued(this);
+ break;
+ case STARTED:
+ l.onJobStarted(this);
+ break;
+ case CANCELLED:
+ l.onJobCancelled(this);
+ break;
+ case FAILED:
+ l.onJobFailed(this, error());
+ break;
+ case SUCCEEDED:
+ try {
+ l.onJobSucceeded(this, result());
+ } catch (Exception e) {
+ // Shouldn't really happen.
+ throw new IllegalStateException(e);
+ }
+ break;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/org/apache/livy/client/common/BufferUtils.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/org/apache/livy/client/common/BufferUtils.java b/client-common/src/main/java/org/apache/livy/client/common/BufferUtils.java
new file mode 100644
index 0000000..55f434d
--- /dev/null
+++ b/client-common/src/main/java/org/apache/livy/client/common/BufferUtils.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.nio.ByteBuffer;
+
+import org.apache.livy.annotations.Private;
+
+/**
+ * Utility methods for dealing with byte buffers and byte arrays.
+ */
+@Private
+public class BufferUtils {
+
+ public static byte[] toByteArray(ByteBuffer buf) {
+ byte[] bytes;
+ if (buf.hasArray() && buf.arrayOffset() == 0 &&
+ buf.remaining() == buf.array().length) {
+ bytes = buf.array();
+ } else {
+ bytes = new byte[buf.remaining()];
+ buf.get(bytes);
+ }
+ return bytes;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/org/apache/livy/client/common/ClientConf.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/org/apache/livy/client/common/ClientConf.java b/client-common/src/main/java/org/apache/livy/client/common/ClientConf.java
new file mode 100644
index 0000000..52fcb6b
--- /dev/null
+++ b/client-common/src/main/java/org/apache/livy/client/common/ClientConf.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.annotations.Private;
+
+/**
+ * Base class with common functionality for type-safe configuration objects.
+ */
+@Private
+public abstract class ClientConf<T extends ClientConf>
+ implements Iterable<Map.Entry<String, String>> {
+
+ protected Logger LOG = LoggerFactory.getLogger(getClass());
+
+ public static interface ConfEntry {
+
+ /** The key in the configuration file. */
+ String key();
+
+ /**
+ * The default value, which also defines the type of the config. Supported types:
+ * Boolean, Integer, Long, String. <code>null</code> maps to String.
+ */
+ Object dflt();
+
+ }
+
+ private static final Map<String, TimeUnit> TIME_SUFFIXES;
+
+ public static final boolean TEST_MODE = Boolean.parseBoolean(System.getenv("LIVY_TEST"));
+
+ static {
+ TIME_SUFFIXES = new HashMap<>();
+ TIME_SUFFIXES.put("us", TimeUnit.MICROSECONDS);
+ TIME_SUFFIXES.put("ms", TimeUnit.MILLISECONDS);
+ TIME_SUFFIXES.put("s", TimeUnit.SECONDS);
+ TIME_SUFFIXES.put("m", TimeUnit.MINUTES);
+ TIME_SUFFIXES.put("min", TimeUnit.MINUTES);
+ TIME_SUFFIXES.put("h", TimeUnit.HOURS);
+ TIME_SUFFIXES.put("d", TimeUnit.DAYS);
+ }
+
+ protected final ConcurrentMap<String, String> config;
+
+ protected ClientConf(Properties config) {
+ this.config = new ConcurrentHashMap<>();
+ if (config != null) {
+ for (String key : config.stringPropertyNames()) {
+ logDeprecationWarning(key);
+ this.config.put(key, config.getProperty(key));
+ }
+ }
+ }
+
+ public String get(String key) {
+ String val = config.get(key);
+ if (val != null) {
+ return val;
+ }
+ DeprecatedConf depConf = getConfigsWithAlternatives().get(key);
+ if (depConf != null) {
+ return config.get(depConf.key());
+ } else {
+ return val;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public T set(String key, String value) {
+ logDeprecationWarning(key);
+ config.put(key, value);
+ return (T) this;
+ }
+
+ @SuppressWarnings("unchecked")
+ public T setIfMissing(String key, String value) {
+ if (config.putIfAbsent(key, value) == null) {
+ logDeprecationWarning(key);
+ }
+ return (T) this;
+ }
+
+ @SuppressWarnings("unchecked")
+ public T setAll(ClientConf<?> other) {
+ for (Map.Entry<String, String> e : other) {
+ set(e.getKey(), e.getValue());
+ }
+ return (T) this;
+ }
+
+ public String get(ConfEntry e) {
+ Object value = get(e, String.class);
+ return (String) (value != null ? value : e.dflt());
+ }
+
+ public boolean getBoolean(ConfEntry e) {
+ String val = get(e, Boolean.class);
+ if (val != null) {
+ return Boolean.parseBoolean(val);
+ } else {
+ return (Boolean) e.dflt();
+ }
+ }
+
+ public int getInt(ConfEntry e) {
+ String val = get(e, Integer.class);
+ if (val != null) {
+ return Integer.parseInt(val);
+ } else {
+ return (Integer) e.dflt();
+ }
+ }
+
+ public long getLong(ConfEntry e) {
+ String val = get(e, Long.class);
+ if (val != null) {
+ return Long.parseLong(val);
+ } else {
+ return (Long) e.dflt();
+ }
+ }
+
+ public long getTimeAsMs(ConfEntry e) {
+ String time = get(e, String.class);
+ if (time == null) {
+ check(e.dflt() != null,
+ "ConfEntry %s doesn't have a default value, cannot convert to time value.", e.key());
+ time = (String) e.dflt();
+ }
+
+ Matcher m = Pattern.compile("(-?[0-9]+)([a-z]+)?").matcher(time.toLowerCase());
+ if (!m.matches()) {
+ throw new IllegalArgumentException("Invalid time string: " + time);
+ }
+
+ long val = Long.parseLong(m.group(1));
+ String suffix = m.group(2);
+
+ if (suffix != null && !TIME_SUFFIXES.containsKey(suffix)) {
+ throw new IllegalArgumentException("Invalid suffix: \"" + suffix + "\"");
+ }
+
+ return TimeUnit.MILLISECONDS.convert(val,
+ suffix != null ? TIME_SUFFIXES.get(suffix) : TimeUnit.MILLISECONDS);
+ }
+
+ @SuppressWarnings("unchecked")
+ public T set(ConfEntry e, Object value) {
+ check(typesMatch(value, e.dflt()), "Value doesn't match configuration entry type for %s.",
+ e.key());
+ if (value == null) {
+ config.remove(e.key());
+ } else {
+ logDeprecationWarning(e.key());
+ config.put(e.key(), value.toString());
+ }
+ return (T) this;
+ }
+
+ @Override
+ public Iterator<Map.Entry<String, String>> iterator() {
+ return config.entrySet().iterator();
+ }
+
+ private String get(ConfEntry e, Class<?> requestedType) {
+ check(getType(e.dflt()).equals(requestedType), "Invalid type conversion requested for %s.",
+ e.key());
+ return this.get(e.key());
+ }
+
+ private boolean typesMatch(Object test, Object expected) {
+ return test == null || getType(test).equals(getType(expected));
+ }
+
+ private Class<?> getType(Object o) {
+ return (o != null) ? o.getClass() : String.class;
+ }
+
+ private void check(boolean test, String message, Object... args) {
+ if (!test) {
+ throw new IllegalArgumentException(String.format(message, args));
+ }
+ }
+
+ /** Logs a warning message if the given config key is deprecated. */
+ private void logDeprecationWarning(String key) {
+ DeprecatedConf altConfs = getConfigsWithAlternatives().get(key);
+ if (altConfs != null) {
+ LOG.warn("The configuration key " + altConfs.key() + " has been deprecated as of Livy "
+ + altConfs.version() + " and may be removed in the future. Please use the new key "
+ + key + " instead.");
+ return;
+ }
+
+ DeprecatedConf depConfs = getDeprecatedConfigs().get(key);
+ if (depConfs != null) {
+ LOG.warn("The configuration key " + depConfs.key() + " has been deprecated as of Livy "
+ + depConfs.version() + " and may be removed in the future. "
+ + depConfs.deprecationMessage());
+ }
+ }
+
+ /** Maps valid key to DeprecatedConf with the deprecated key. */
+ protected abstract Map<String, DeprecatedConf> getConfigsWithAlternatives();
+
+ /** Maps deprecated key to DeprecatedConf with the same key. */
+ protected abstract Map<String, DeprecatedConf> getDeprecatedConfigs();
+
+ public static interface DeprecatedConf {
+
+ /** The key in the configuration file. */
+ String key();
+
+ /** The Livy version in which the key was deprecated. */
+ String version();
+
+ /** Message to include in the deprecation warning for configs without alternatives */
+ String deprecationMessage();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/org/apache/livy/client/common/HttpMessages.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/org/apache/livy/client/common/HttpMessages.java b/client-common/src/main/java/org/apache/livy/client/common/HttpMessages.java
new file mode 100644
index 0000000..99ce900
--- /dev/null
+++ b/client-common/src/main/java/org/apache/livy/client/common/HttpMessages.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.livy.JobHandle.State;
+import org.apache.livy.annotations.Private;
+
+/**
+ * There are the Java representations of the JSON messages used by the client protocol.
+ *
+ * Note that Jackson requires an empty constructor (or annotations) to be able to instantiate
+ * types, so the extra noise is necessary here.
+ */
+@Private
+public class HttpMessages {
+
+ public static interface ClientMessage {
+
+ }
+
+ public static class CreateClientRequest implements ClientMessage {
+
+ public final Map<String, String> conf;
+
+ public CreateClientRequest(Map<String, String> conf) {
+ this.conf = conf;
+ }
+
+ private CreateClientRequest() {
+ this(null);
+ }
+
+ }
+
+ public static class SessionInfo implements ClientMessage {
+
+ public final int id;
+ public final String appId;
+ public final String owner;
+ public final String proxyUser;
+ public final String state;
+ public final String kind;
+ public final Map<String, String> appInfo;
+ public final List<String> log;
+
+ public SessionInfo(int id, String appId, String owner, String proxyUser, String state,
+ String kind, Map<String, String> appInfo, List<String> log) {
+ this.id = id;
+ this.appId = appId;
+ this.owner = owner;
+ this.proxyUser = proxyUser;
+ this.state = state;
+ this.kind = kind;
+ this.appInfo = appInfo;
+ this.log = log;
+ }
+
+ private SessionInfo() {
+ this(-1, null, null, null, null, null, null, null);
+ }
+
+ }
+
+ public static class SerializedJob implements ClientMessage {
+
+ public final byte[] job;
+
+ public SerializedJob(byte[] job) {
+ this.job = job;
+ }
+
+ private SerializedJob() {
+ this(null);
+ }
+
+ }
+
+ public static class AddResource implements ClientMessage {
+
+ public final String uri;
+
+ public AddResource(String uri) {
+ this.uri = uri;
+ }
+
+ private AddResource() {
+ this(null);
+ }
+
+ }
+
+ public static class JobStatus implements ClientMessage {
+
+ public final long id;
+ public final State state;
+ public final byte[] result;
+ public final String error;
+
+ public JobStatus(long id, State state, byte[] result, String error) {
+ this.id = id;
+ this.state = state;
+ this.error = error;
+
+ // json4s, at least, seems confused about whether a "null" in the JSON payload should
+ // become a null array or a byte array with length 0. Since there shouldn't be any
+ // valid serialized object in a byte array of size 0, translate that to null.
+ this.result = (result != null && result.length > 0) ? result : null;
+
+ if (this.result != null && state != State.SUCCEEDED) {
+ throw new IllegalArgumentException("Result cannot be set unless job succeeded.");
+ }
+ // The check for "result" is not completely correct, but is here to make the unit tests work.
+ if (this.result == null && error != null && state != State.FAILED) {
+ throw new IllegalArgumentException("Error cannot be set unless job failed.");
+ }
+ }
+
+ private JobStatus() {
+ this(-1, null, null, null);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/org/apache/livy/client/common/Serializer.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/org/apache/livy/client/common/Serializer.java b/client-common/src/main/java/org/apache/livy/client/common/Serializer.java
new file mode 100644
index 0000000..3ea3f56
--- /dev/null
+++ b/client-common/src/main/java/org/apache/livy/client/common/Serializer.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import com.esotericsoftware.shaded.org.objenesis.strategy.StdInstantiatorStrategy;
+
+import org.apache.livy.annotations.Private;
+
+/**
+ * Utility class to serialize user data using Kryo.
+ */
+@Private
+public class Serializer {
+
+ // Kryo docs say 0-8 are taken. Strange things happen if you don't set an ID when registering
+ // classes.
+ private static final int REG_ID_BASE = 16;
+
+ private final ThreadLocal<Kryo> kryos;
+
+ public Serializer(final Class<?>... klasses) {
+ this.kryos = new ThreadLocal<Kryo>() {
+ @Override
+ protected Kryo initialValue() {
+ Kryo kryo = new Kryo();
+ int count = 0;
+ for (Class<?> klass : klasses) {
+ kryo.register(klass, REG_ID_BASE + count);
+ count++;
+ }
+ kryo.setInstantiatorStrategy(new StdInstantiatorStrategy());
+ kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
+ return kryo;
+ }
+ };
+ }
+
+ public Object deserialize(ByteBuffer data) {
+ byte[] b = new byte[data.remaining()];
+ data.get(b);
+ Input kryoIn = new Input(b);
+ return kryos.get().readClassAndObject(kryoIn);
+ }
+
+ public ByteBuffer serialize(Object data) {
+ ByteBufferOutputStream out = new ByteBufferOutputStream();
+ Output kryoOut = new Output(out);
+ kryos.get().writeClassAndObject(kryoOut, data);
+ kryoOut.flush();
+ return out.getBuffer();
+ }
+
+ private static class ByteBufferOutputStream extends ByteArrayOutputStream {
+
+ public ByteBuffer getBuffer() {
+ ByteBuffer result = ByteBuffer.wrap(buf, 0, count);
+ buf = null;
+ reset();
+ return result;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/main/java/org/apache/livy/client/common/TestUtils.java
----------------------------------------------------------------------
diff --git a/client-common/src/main/java/org/apache/livy/client/common/TestUtils.java b/client-common/src/main/java/org/apache/livy/client/common/TestUtils.java
new file mode 100644
index 0000000..e1e450c
--- /dev/null
+++ b/client-common/src/main/java/org/apache/livy/client/common/TestUtils.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.livy.annotations.Private;
+
+/**
+ * Utility methods used by Livy tests.
+ */
+@Private
+public class TestUtils {
+
+ /**
+ * Returns JVM arguments that enable jacoco on a process to be run. The returned arguments
+ * create a new, unique output file in the same directory referenced by the "jacoco.args"
+ * system property.
+ *
+ * @return JVM arguments, or null.
+ */
+ public static String getJacocoArgs() {
+ String jacocoArgs = System.getProperty("jacoco.args");
+ if (jacocoArgs == null) {
+ return null;
+ }
+
+ Pattern p = Pattern.compile("(.+?destfile=)(.+?)(,.+)?");
+ Matcher m = p.matcher(jacocoArgs);
+ if (!m.matches()) {
+ return null;
+ }
+
+ String fileName = new File(m.group(2)).getName();
+ File outputDir = new File(m.group(2)).getParentFile();
+
+ File newFile;
+ while (true) {
+ int newId = outputDir.list().length;
+ newFile = new File(outputDir, "jacoco-" + newId + ".exec");
+ try {
+ Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW).close();
+ break;
+ } catch (IOException ioe) {
+ // Try again.
+ }
+ }
+
+ StringBuilder newArgs = new StringBuilder();
+ newArgs.append(m.group(1));
+ newArgs.append(newFile.getAbsolutePath());
+ if (m.group(3) != null) {
+ newArgs.append(m.group(3));
+ }
+
+ return newArgs.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/com/cloudera/livy/client/common/TestAbstractJobHandle.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/com/cloudera/livy/client/common/TestAbstractJobHandle.java b/client-common/src/test/java/com/cloudera/livy/client/common/TestAbstractJobHandle.java
deleted file mode 100644
index 1b48f07..0000000
--- a/client-common/src/test/java/com/cloudera/livy/client/common/TestAbstractJobHandle.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.util.concurrent.TimeUnit;
-
-import org.junit.Test;
-import org.mockito.InOrder;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
-
-import com.cloudera.livy.JobHandle;
-import com.cloudera.livy.JobHandle.State;
-
-public class TestAbstractJobHandle {
-
- @Test
- public void testJobHandle() {
- AbstractJobHandle<Void> handle = new TestJobHandle();
-
- assertTrue(handle.changeState(State.QUEUED));
- assertEquals(State.QUEUED, handle.getState());
-
- @SuppressWarnings("unchecked")
- JobHandle.Listener<Void> l1 = mock(JobHandle.Listener.class);
- handle.addListener(l1);
- verify(l1).onJobQueued(handle);
-
- assertTrue(handle.changeState(State.STARTED));
- verify(l1).onJobStarted(handle);
-
- assertTrue(handle.changeState(State.SUCCEEDED));
- verify(l1).onJobSucceeded(handle, null);
-
- assertFalse(handle.changeState(State.CANCELLED));
- }
-
- private static class TestJobHandle extends AbstractJobHandle<Void> {
-
- @Override
- protected Void result() {
- return null;
- }
-
- @Override
- protected Throwable error() {
- return null;
- }
-
- @Override
- public Void get() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Void get(long l, TimeUnit t) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean isDone() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean isCancelled() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean cancel(boolean b) {
- throw new UnsupportedOperationException();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/com/cloudera/livy/client/common/TestBufferUtils.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/com/cloudera/livy/client/common/TestBufferUtils.java b/client-common/src/test/java/com/cloudera/livy/client/common/TestBufferUtils.java
deleted file mode 100644
index 725d506..0000000
--- a/client-common/src/test/java/com/cloudera/livy/client/common/TestBufferUtils.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-public class TestBufferUtils {
-
- @Test
- public void testWrappedArray() {
- byte[] array = new byte[] { 0x1b, 0x2b };
- byte[] unwrapped = BufferUtils.toByteArray(ByteBuffer.wrap(array));
- assertSame(array, unwrapped);
- }
-
- @Test
- public void testShortArray() {
- byte[] array = new byte[] { 0x1b, 0x2b };
- byte[] unwrapped = BufferUtils.toByteArray(ByteBuffer.wrap(array, 0, 1));
- assertNotSame(array, unwrapped);
- assertEquals(1, unwrapped.length);
- }
-
- @Test
- public void testOffsetArray() {
- byte[] array = new byte[] { 0x1b, 0x2b };
- byte[] unwrapped = BufferUtils.toByteArray(ByteBuffer.wrap(array, 1, 1));
- assertNotSame(array, unwrapped);
- assertEquals(1, unwrapped.length);
- }
-
- @Test
- public void testDirectBuffer() {
- ByteBuffer direct = ByteBuffer.allocateDirect(1);
- direct.put((byte) 0x1b);
- assertFalse(direct.hasArray());
- direct.flip();
-
- byte[] unwrapped = BufferUtils.toByteArray(direct);
- assertEquals(0x1b, unwrapped[0]);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/com/cloudera/livy/client/common/TestClientConf.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/com/cloudera/livy/client/common/TestClientConf.java b/client-common/src/test/java/com/cloudera/livy/client/common/TestClientConf.java
deleted file mode 100644
index f00c996..0000000
--- a/client-common/src/test/java/com/cloudera/livy/client/common/TestClientConf.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-public class TestClientConf {
-
- @Test
- public void testTypes() {
- TestConf conf = new TestConf(null);
-
- assertNull(conf.get(TestConf.Entry.NULL));
- assertEquals("default", conf.get(TestConf.Entry.STRING));
- assertEquals(false, conf.getBoolean(TestConf.Entry.BOOLEAN));
- assertEquals(42, conf.getInt(TestConf.Entry.INT));
- assertEquals(84L, conf.getLong(TestConf.Entry.LONG));
- assertEquals(168L, conf.getTimeAsMs(TestConf.Entry.TIME));
-
- try {
- conf.get(TestConf.Entry.INT);
- fail("Should have failed to retrieve int as string.");
- } catch (IllegalArgumentException ie) {
- // Expected.
- }
-
- conf.set(TestConf.Entry.INT, 336);
- assertEquals(336, conf.getInt(TestConf.Entry.INT));
-
- try {
- conf.set(TestConf.Entry.INT, "abcde");
- fail("Should have failed to set int as string.");
- } catch (IllegalArgumentException ie) {
- // Expected.
- }
-
- conf.set(TestConf.Entry.STRING, "aString");
- assertEquals("aString", conf.get(TestConf.Entry.STRING));
-
- conf.set(TestConf.Entry.BOOLEAN, true);
- assertEquals(true, conf.getBoolean(TestConf.Entry.BOOLEAN));
-
- conf.set(TestConf.Entry.LONG, 42L);
- assertEquals(42L, conf.getLong(TestConf.Entry.LONG));
-
- conf.set(TestConf.Entry.LONG, null);
- assertEquals(84L, conf.getLong(TestConf.Entry.LONG));
-
- conf.set(TestConf.Entry.TIME_NO_DEFAULT, "100");
- assertEquals(100L, conf.getTimeAsMs(TestConf.Entry.TIME_NO_DEFAULT));
- }
-
- @Test
- public void testRawProperties() {
- Properties dflt = new Properties();
- dflt.put("key1", "val1");
- dflt.put("key2", "val2");
-
- TestConf conf = new TestConf(dflt);
- conf.set("key2", "anotherVal");
-
- assertEquals("val1", conf.get("key1"));
- assertEquals("anotherVal", conf.get("key2"));
-
- conf.setIfMissing("key2", "yetAnotherVal");
- assertEquals("anotherVal", conf.get("key2"));
-
- conf.setIfMissing("key3", "val3");
- assertEquals("val3", conf.get("key3"));
-
- int count = 0;
- for (Map.Entry<String, String> e : conf) {
- count++;
- }
- assertEquals(3, count);
-
- TestConf newProps = new TestConf(null);
- newProps.set("key4", "val4");
- newProps.set("key5", "val5");
- conf.setAll(newProps);
- assertEquals("val4", conf.get("key4"));
- assertEquals("val5", conf.get("key5"));
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testInvalidTime() {
- TestConf conf = new TestConf(null);
- conf.set(TestConf.Entry.TIME, "invalid");
- conf.getTimeAsMs(TestConf.Entry.TIME);
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testInvalidTimeSuffix() {
- TestConf conf = new TestConf(null);
- conf.set(TestConf.Entry.TIME, "100foo");
- conf.getTimeAsMs(TestConf.Entry.TIME);
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testTimeWithoutDefault() {
- TestConf conf = new TestConf(null);
- conf.getTimeAsMs(TestConf.Entry.TIME_NO_DEFAULT);
- }
-
-
- @Test
- public void testDeprecation() {
- TestConf conf = new TestConf(null);
-
- assertNull(conf.get("depKey"));
- assertNull(conf.get("dep_alt"));
- assertNull(conf.get("new-key"));
- assertEquals("value", conf.get(TestConf.Entry.NEW_CONF));
-
- TestConf depProps = new TestConf(null);
- depProps.set("depKey", "dep-val");
- depProps.set("dep_alt", "alt-val");
- conf.setAll(depProps);
- assertEquals("dep-val", conf.get("depKey"));
- assertEquals("alt-val", conf.get("dep_alt"));
- assertEquals("alt-val", conf.get(TestConf.Entry.NEW_CONF));
- assertEquals("alt-val", conf.get("new-key"));
-
- conf.set("new-key", "new-val");
- assertEquals("new-val", conf.get(TestConf.Entry.NEW_CONF));
- assertEquals("alt-val", conf.get("dep_alt"));
- assertEquals("new-val", conf.get("new-key"));
- }
-
- private static class TestConf extends ClientConf<TestConf> {
-
- static enum Entry implements ConfEntry {
- NULL("null", null),
- STRING("string", "default"),
- BOOLEAN("bool", false),
- INT("int", 42),
- LONG("long", 84L),
- TIME("time", "168ms"),
- TIME_NO_DEFAULT("time2", null),
- NEW_CONF("new-key", "value");
-
- private final String key;
- private final Object dflt;
-
- private Entry(String key, Object dflt) {
- this.key = key;
- this.dflt = dflt;
- }
-
- @Override
- public String key() { return key; }
-
- @Override
- public Object dflt() { return dflt; }
-
- }
-
- TestConf(Properties p) {
- super(p);
- }
-
- private static final Map<String, DeprecatedConf> configsWithAlternatives
- = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{
- put(TestConf.Entry.NEW_CONF.key, DepConf.DEP_WITH_ALT);
- }});
-
- private static final Map<String, DeprecatedConf> deprecatedConfigs
- = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{
- put(DepConf.DEP_NO_ALT.key, DepConf.DEP_NO_ALT);
- }});
-
- protected Map<String, DeprecatedConf> getConfigsWithAlternatives() {
- return configsWithAlternatives;
- }
-
- protected Map<String, DeprecatedConf> getDeprecatedConfigs() {
- return deprecatedConfigs;
- }
-
- static enum DepConf implements DeprecatedConf {
- DEP_WITH_ALT("dep_alt", "0.4"),
- DEP_NO_ALT("depKey", "1.0");
-
- private final String key;
- private final String version;
- private final String deprecationMessage;
-
- private DepConf(String key, String version) {
- this(key, version, "");
- }
-
- private DepConf(String key, String version, String deprecationMessage) {
- this.key = key;
- this.version = version;
- this.deprecationMessage = deprecationMessage;
- }
-
- @Override
- public String key() { return key; }
-
- @Override
- public String version() { return version; }
-
- @Override
- public String deprecationMessage() { return deprecationMessage; }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/com/cloudera/livy/client/common/TestHttpMessages.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/com/cloudera/livy/client/common/TestHttpMessages.java b/client-common/src/test/java/com/cloudera/livy/client/common/TestHttpMessages.java
deleted file mode 100644
index c6adeca..0000000
--- a/client-common/src/test/java/com/cloudera/livy/client/common/TestHttpMessages.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Field;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-import com.cloudera.livy.JobHandle.State;
-
-public class TestHttpMessages {
-
- /**
- * Tests that all defined messages can be serialized and deserialized using Jackson.
- */
- @Test
- public void testMessageSerialization() throws Exception {
- ObjectMapper mapper = new ObjectMapper();
-
- for (Class<?> msg : HttpMessages.class.getClasses()) {
- if (msg.isInterface()) {
- continue;
- }
-
- String name = msg.getSimpleName();
-
- Constructor c = msg.getConstructors()[0];
- Object[] params = new Object[c.getParameterTypes().length];
- Type[] genericTypes = c.getGenericParameterTypes();
- for (int i = 0; i < params.length; i++) {
- params[i] = dummyValue(c.getParameterTypes()[i], genericTypes[i]);
- }
-
- Object o1 = c.newInstance(params);
- byte[] serialized = mapper.writeValueAsBytes(o1);
- Object o2 = mapper.readValue(serialized, msg);
-
- assertNotNull("could not deserialize " + name, o2);
- for (Field f : msg.getFields()) {
- checkEquals(name, f, o1, o2);
- }
- }
-
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testJobStatusResultBadState() {
- new HttpMessages.JobStatus(0L, State.QUEUED, new byte[1], null);
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testJobStatusErrorBadState() {
- new HttpMessages.JobStatus(0L, State.QUEUED, null, "An Error");
- }
-
- private Object dummyValue(Class<?> klass, Type type) {
- switch (klass.getSimpleName()) {
- case "int":
- case "Integer":
- return 42;
- case "long": return 84L;
- case "byte[]": return new byte[] { (byte) 0x42, (byte) 0x84 };
- case "String": return "test";
- case "State": return State.SUCCEEDED;
- case "Map":
- Map<String, String> map = new HashMap<>();
- map.put("dummy1", "dummy2");
- return map;
- case "List":
- Class<?> genericType = getGenericArgType(type);
- return Arrays.asList(dummyValue(genericType, null), dummyValue(genericType, null));
- default: throw new IllegalArgumentException("FIX ME: " + klass.getSimpleName());
- }
- }
-
- private Class<?> getGenericArgType(Type type) {
- assertNotNull("FIX ME: null type argument.", type);
-
- ParameterizedType ptype = (ParameterizedType) type;
- assertEquals("FIX ME: no support for multiple type arguments.",
- 1, ptype.getActualTypeArguments().length);
-
- Type argType = ptype.getActualTypeArguments()[0];
- assertTrue("FIX ME: type argument is not a class.", argType instanceof Class);
-
- return (Class<?>) argType;
- }
-
- private void checkEquals(String name, Field f, Object o1, Object o2) throws Exception {
- Object v1 = f.get(o1);
- Object v2 = f.get(o2);
-
- boolean match;
- if (!f.getType().isArray()) {
- match = v1.equals(v2);
- } else if (v1 instanceof byte[]) {
- match = Arrays.equals((byte[]) v1, (byte[]) v2);
- } else {
- throw new IllegalArgumentException("FIX ME: " + f.getType().getSimpleName());
- }
-
- assertTrue(
- String.format("Field %s of %s does not match after deserialization.", f.getName(), name),
- match);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/com/cloudera/livy/client/common/TestSerializer.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/com/cloudera/livy/client/common/TestSerializer.java b/client-common/src/test/java/com/cloudera/livy/client/common/TestSerializer.java
deleted file mode 100644
index 76fade1..0000000
--- a/client-common/src/test/java/com/cloudera/livy/client/common/TestSerializer.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.List;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-public class TestSerializer {
-
- private static final String MESSAGE = "Hello World!";
-
- @Test
- public void testSerializer() throws Exception {
- Object decoded = doSerDe(MESSAGE);
- assertEquals(MESSAGE, decoded);
- }
-
- @Test
- public void testUnicodeSerializer() throws Exception {
- StringBuilder builder = new StringBuilder();
- for (int x = 0; x < 5000; x++) {
- builder.append("\u263A");
- }
- String testMessage = builder.toString();
- Object decoded = doSerDe(testMessage);
- assertEquals(testMessage, decoded);
- }
-
- @Test
- public void testAutoRegistration() throws Exception {
- Object decoded = doSerDe(new TestMessage(MESSAGE), TestMessage.class);
- assertTrue(decoded instanceof TestMessage);
- assertEquals(MESSAGE, ((TestMessage)decoded).data);
- }
-
- private Object doSerDe(Object data, Class<?>... klasses) {
- Serializer s = new Serializer(klasses);
- ByteBuffer serialized = s.serialize(data);
- return s.deserialize(serialized);
- }
-
- private ByteBuffer newBuffer() {
- return ByteBuffer.allocate(1024);
- }
-
- private static class TestMessage {
- final String data;
-
- TestMessage() {
- this(null);
- }
-
- TestMessage(String data) {
- this.data = data;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/com/cloudera/livy/client/common/TestTestUtils.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/com/cloudera/livy/client/common/TestTestUtils.java b/client-common/src/test/java/com/cloudera/livy/client/common/TestTestUtils.java
deleted file mode 100644
index de1bf93..0000000
--- a/client-common/src/test/java/com/cloudera/livy/client/common/TestTestUtils.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.common;
-
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-public class TestTestUtils {
-
- @Test
- public void testJacocoArgs() {
- String args1 = TestUtils.getJacocoArgs();
- String expected1 = System.getProperty("jacoco.args").replace("main.exec", "jacoco-1.exec");
- assertEquals(expected1, args1);
-
- String args2 = TestUtils.getJacocoArgs();
- String expected2 = System.getProperty("jacoco.args").replace("main.exec", "jacoco-2.exec");
- assertEquals(expected2, args2);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/org/apache/livy/client/common/TestAbstractJobHandle.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/org/apache/livy/client/common/TestAbstractJobHandle.java b/client-common/src/test/java/org/apache/livy/client/common/TestAbstractJobHandle.java
new file mode 100644
index 0000000..6703147
--- /dev/null
+++ b/client-common/src/test/java/org/apache/livy/client/common/TestAbstractJobHandle.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+import org.mockito.InOrder;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import org.apache.livy.JobHandle;
+import org.apache.livy.JobHandle.State;
+
+public class TestAbstractJobHandle {
+
+ @Test
+ public void testJobHandle() {
+ AbstractJobHandle<Void> handle = new TestJobHandle();
+
+ assertTrue(handle.changeState(State.QUEUED));
+ assertEquals(State.QUEUED, handle.getState());
+
+ @SuppressWarnings("unchecked")
+ JobHandle.Listener<Void> l1 = mock(JobHandle.Listener.class);
+ handle.addListener(l1);
+ verify(l1).onJobQueued(handle);
+
+ assertTrue(handle.changeState(State.STARTED));
+ verify(l1).onJobStarted(handle);
+
+ assertTrue(handle.changeState(State.SUCCEEDED));
+ verify(l1).onJobSucceeded(handle, null);
+
+ assertFalse(handle.changeState(State.CANCELLED));
+ }
+
+ private static class TestJobHandle extends AbstractJobHandle<Void> {
+
+ @Override
+ protected Void result() {
+ return null;
+ }
+
+ @Override
+ protected Throwable error() {
+ return null;
+ }
+
+ @Override
+ public Void get() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Void get(long l, TimeUnit t) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isDone() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isCancelled() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean cancel(boolean b) {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/org/apache/livy/client/common/TestBufferUtils.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/org/apache/livy/client/common/TestBufferUtils.java b/client-common/src/test/java/org/apache/livy/client/common/TestBufferUtils.java
new file mode 100644
index 0000000..b41111a
--- /dev/null
+++ b/client-common/src/test/java/org/apache/livy/client/common/TestBufferUtils.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestBufferUtils {
+
+ @Test
+ public void testWrappedArray() {
+ byte[] array = new byte[] { 0x1b, 0x2b };
+ byte[] unwrapped = BufferUtils.toByteArray(ByteBuffer.wrap(array));
+ assertSame(array, unwrapped);
+ }
+
+ @Test
+ public void testShortArray() {
+ byte[] array = new byte[] { 0x1b, 0x2b };
+ byte[] unwrapped = BufferUtils.toByteArray(ByteBuffer.wrap(array, 0, 1));
+ assertNotSame(array, unwrapped);
+ assertEquals(1, unwrapped.length);
+ }
+
+ @Test
+ public void testOffsetArray() {
+ byte[] array = new byte[] { 0x1b, 0x2b };
+ byte[] unwrapped = BufferUtils.toByteArray(ByteBuffer.wrap(array, 1, 1));
+ assertNotSame(array, unwrapped);
+ assertEquals(1, unwrapped.length);
+ }
+
+ @Test
+ public void testDirectBuffer() {
+ ByteBuffer direct = ByteBuffer.allocateDirect(1);
+ direct.put((byte) 0x1b);
+ assertFalse(direct.hasArray());
+ direct.flip();
+
+ byte[] unwrapped = BufferUtils.toByteArray(direct);
+ assertEquals(0x1b, unwrapped[0]);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/org/apache/livy/client/common/TestClientConf.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/org/apache/livy/client/common/TestClientConf.java b/client-common/src/test/java/org/apache/livy/client/common/TestClientConf.java
new file mode 100644
index 0000000..02bbaaa
--- /dev/null
+++ b/client-common/src/test/java/org/apache/livy/client/common/TestClientConf.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestClientConf {
+
+ @Test
+ public void testTypes() {
+ TestConf conf = new TestConf(null);
+
+ assertNull(conf.get(TestConf.Entry.NULL));
+ assertEquals("default", conf.get(TestConf.Entry.STRING));
+ assertEquals(false, conf.getBoolean(TestConf.Entry.BOOLEAN));
+ assertEquals(42, conf.getInt(TestConf.Entry.INT));
+ assertEquals(84L, conf.getLong(TestConf.Entry.LONG));
+ assertEquals(168L, conf.getTimeAsMs(TestConf.Entry.TIME));
+
+ try {
+ conf.get(TestConf.Entry.INT);
+ fail("Should have failed to retrieve int as string.");
+ } catch (IllegalArgumentException ie) {
+ // Expected.
+ }
+
+ conf.set(TestConf.Entry.INT, 336);
+ assertEquals(336, conf.getInt(TestConf.Entry.INT));
+
+ try {
+ conf.set(TestConf.Entry.INT, "abcde");
+ fail("Should have failed to set int as string.");
+ } catch (IllegalArgumentException ie) {
+ // Expected.
+ }
+
+ conf.set(TestConf.Entry.STRING, "aString");
+ assertEquals("aString", conf.get(TestConf.Entry.STRING));
+
+ conf.set(TestConf.Entry.BOOLEAN, true);
+ assertEquals(true, conf.getBoolean(TestConf.Entry.BOOLEAN));
+
+ conf.set(TestConf.Entry.LONG, 42L);
+ assertEquals(42L, conf.getLong(TestConf.Entry.LONG));
+
+ conf.set(TestConf.Entry.LONG, null);
+ assertEquals(84L, conf.getLong(TestConf.Entry.LONG));
+
+ conf.set(TestConf.Entry.TIME_NO_DEFAULT, "100");
+ assertEquals(100L, conf.getTimeAsMs(TestConf.Entry.TIME_NO_DEFAULT));
+ }
+
+ @Test
+ public void testRawProperties() {
+ Properties dflt = new Properties();
+ dflt.put("key1", "val1");
+ dflt.put("key2", "val2");
+
+ TestConf conf = new TestConf(dflt);
+ conf.set("key2", "anotherVal");
+
+ assertEquals("val1", conf.get("key1"));
+ assertEquals("anotherVal", conf.get("key2"));
+
+ conf.setIfMissing("key2", "yetAnotherVal");
+ assertEquals("anotherVal", conf.get("key2"));
+
+ conf.setIfMissing("key3", "val3");
+ assertEquals("val3", conf.get("key3"));
+
+ int count = 0;
+ for (Map.Entry<String, String> e : conf) {
+ count++;
+ }
+ assertEquals(3, count);
+
+ TestConf newProps = new TestConf(null);
+ newProps.set("key4", "val4");
+ newProps.set("key5", "val5");
+ conf.setAll(newProps);
+ assertEquals("val4", conf.get("key4"));
+ assertEquals("val5", conf.get("key5"));
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testInvalidTime() {
+ TestConf conf = new TestConf(null);
+ conf.set(TestConf.Entry.TIME, "invalid");
+ conf.getTimeAsMs(TestConf.Entry.TIME);
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testInvalidTimeSuffix() {
+ TestConf conf = new TestConf(null);
+ conf.set(TestConf.Entry.TIME, "100foo");
+ conf.getTimeAsMs(TestConf.Entry.TIME);
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testTimeWithoutDefault() {
+ TestConf conf = new TestConf(null);
+ conf.getTimeAsMs(TestConf.Entry.TIME_NO_DEFAULT);
+ }
+
+
+ @Test
+ public void testDeprecation() {
+ TestConf conf = new TestConf(null);
+
+ assertNull(conf.get("depKey"));
+ assertNull(conf.get("dep_alt"));
+ assertNull(conf.get("new-key"));
+ assertEquals("value", conf.get(TestConf.Entry.NEW_CONF));
+
+ TestConf depProps = new TestConf(null);
+ depProps.set("depKey", "dep-val");
+ depProps.set("dep_alt", "alt-val");
+ conf.setAll(depProps);
+ assertEquals("dep-val", conf.get("depKey"));
+ assertEquals("alt-val", conf.get("dep_alt"));
+ assertEquals("alt-val", conf.get(TestConf.Entry.NEW_CONF));
+ assertEquals("alt-val", conf.get("new-key"));
+
+ conf.set("new-key", "new-val");
+ assertEquals("new-val", conf.get(TestConf.Entry.NEW_CONF));
+ assertEquals("alt-val", conf.get("dep_alt"));
+ assertEquals("new-val", conf.get("new-key"));
+ }
+
+ private static class TestConf extends ClientConf<TestConf> {
+
+ static enum Entry implements ConfEntry {
+ NULL("null", null),
+ STRING("string", "default"),
+ BOOLEAN("bool", false),
+ INT("int", 42),
+ LONG("long", 84L),
+ TIME("time", "168ms"),
+ TIME_NO_DEFAULT("time2", null),
+ NEW_CONF("new-key", "value");
+
+ private final String key;
+ private final Object dflt;
+
+ private Entry(String key, Object dflt) {
+ this.key = key;
+ this.dflt = dflt;
+ }
+
+ @Override
+ public String key() { return key; }
+
+ @Override
+ public Object dflt() { return dflt; }
+
+ }
+
+ TestConf(Properties p) {
+ super(p);
+ }
+
+ private static final Map<String, DeprecatedConf> configsWithAlternatives
+ = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{
+ put(TestConf.Entry.NEW_CONF.key, DepConf.DEP_WITH_ALT);
+ }});
+
+ private static final Map<String, DeprecatedConf> deprecatedConfigs
+ = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{
+ put(DepConf.DEP_NO_ALT.key, DepConf.DEP_NO_ALT);
+ }});
+
+ protected Map<String, DeprecatedConf> getConfigsWithAlternatives() {
+ return configsWithAlternatives;
+ }
+
+ protected Map<String, DeprecatedConf> getDeprecatedConfigs() {
+ return deprecatedConfigs;
+ }
+
+ static enum DepConf implements DeprecatedConf {
+ DEP_WITH_ALT("dep_alt", "0.4"),
+ DEP_NO_ALT("depKey", "1.0");
+
+ private final String key;
+ private final String version;
+ private final String deprecationMessage;
+
+ private DepConf(String key, String version) {
+ this(key, version, "");
+ }
+
+ private DepConf(String key, String version, String deprecationMessage) {
+ this.key = key;
+ this.version = version;
+ this.deprecationMessage = deprecationMessage;
+ }
+
+ @Override
+ public String key() { return key; }
+
+ @Override
+ public String version() { return version; }
+
+ @Override
+ public String deprecationMessage() { return deprecationMessage; }
+ }
+ }
+
+}
[27/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/scala-2.10/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala b/repl/scala-2.10/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala
new file mode 100644
index 0000000..e3c849d
--- /dev/null
+++ b/repl/scala-2.10/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import org.scalatest._
+
+import org.apache.livy.LivyBaseUnitTestSuite
+
+class SparkInterpreterSpec extends FunSpec with Matchers with LivyBaseUnitTestSuite {
+ describe("SparkInterpreter") {
+ val interpreter = new SparkInterpreter(null)
+
+ it("should parse Scala compile error.") {
+ // Regression test for LIVY-260.
+ val error =
+ """<console>:27: error: type mismatch;
+ | found : Int
+ | required: String
+ | sc.setJobGroup(groupName, groupName, true)
+ | ^
+ |<console>:27: error: type mismatch;
+ | found : Int
+ | required: String
+ | sc.setJobGroup(groupName, groupName, true)
+ | ^
+ |""".stripMargin
+
+ val expectedTraceback = AbstractSparkInterpreter.KEEP_NEWLINE_REGEX.split(
+ """ found : Int
+ | required: String
+ | sc.setJobGroup(groupName, groupName, true)
+ | ^
+ |<console>:27: error: type mismatch;
+ | found : Int
+ | required: String
+ | sc.setJobGroup(groupName, groupName, true)
+ | ^
+ |""".stripMargin)
+
+ val (ename, traceback) = interpreter.parseError(error)
+ ename shouldBe "<console>:27: error: type mismatch;"
+ traceback shouldBe expectedTraceback
+ }
+
+ it("should parse Scala runtime error and remove internal frames.") {
+ val error =
+ """java.lang.RuntimeException: message
+ | at $iwC$$iwC$$iwC$$iwC$$iwC.error(<console>:25)
+ | at $iwC$$iwC$$iwC.error2(<console>:27)
+ | at $iwC$$iwC.<init>(<console>:41)
+ | at $iwC.<init>(<console>:43)
+ | at <init>(<console>:45)
+ | at .<init>(<console>:49)
+ | at .<clinit>(<console>)
+ | at .<init>(<console>:7)
+ | at .<clinit>(<console>)
+ | at $print(<console>)
+ | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+ |""".stripMargin
+
+ val expectedTraceback = AbstractSparkInterpreter.KEEP_NEWLINE_REGEX.split(
+ """ at <user code>.error(<console>:25)
+ | at <user code>.error2(<console>:27)
+ |""".stripMargin)
+
+ val (ename, traceback) = interpreter.parseError(error)
+ ename shouldBe "java.lang.RuntimeException: message"
+ traceback shouldBe expectedTraceback
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/scala-2.11/pom.xml
----------------------------------------------------------------------
diff --git a/repl/scala-2.11/pom.xml b/repl/scala-2.11/pom.xml
index 745997c..97f5c54 100644
--- a/repl/scala-2.11/pom.xml
+++ b/repl/scala-2.11/pom.xml
@@ -19,15 +19,15 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-repl_2.11</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-repl-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala b/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
deleted file mode 100644
index 925d477..0000000
--- a/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import java.io.File
-import java.net.URLClassLoader
-import java.nio.file.{Files, Paths}
-
-import scala.tools.nsc.Settings
-import scala.tools.nsc.interpreter.JPrintWriter
-import scala.tools.nsc.interpreter.Results.Result
-import scala.util.control.NonFatal
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.repl.SparkILoop
-
-/**
- * Scala 2.11 version of SparkInterpreter
- */
-class SparkInterpreter(conf: SparkConf)
- extends AbstractSparkInterpreter with SparkContextInitializer {
-
- protected var sparkContext: SparkContext = _
- private var sparkILoop: SparkILoop = _
- private var sparkHttpServer: Object = _
-
- override def start(): SparkContext = {
- require(sparkILoop == null)
-
- val rootDir = conf.get("spark.repl.classdir", System.getProperty("java.io.tmpdir"))
- val outputDir = Files.createTempDirectory(Paths.get(rootDir), "spark").toFile
- outputDir.deleteOnExit()
- conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath)
-
- // Only Spark1 requires to create http server, Spark2 removes HttpServer class.
- startHttpServer(outputDir).foreach { case (server, uri) =>
- sparkHttpServer = server
- conf.set("spark.repl.class.uri", uri)
- }
-
- val settings = new Settings()
- settings.processArguments(List("-Yrepl-class-based",
- "-Yrepl-outdir", s"${outputDir.getAbsolutePath}"), true)
- settings.usejavacp.value = true
- settings.embeddedDefaults(Thread.currentThread().getContextClassLoader())
-
- sparkILoop = new SparkILoop(None, new JPrintWriter(outputStream, true))
- sparkILoop.settings = settings
- sparkILoop.createInterpreter()
- sparkILoop.initializeSynchronous()
-
- restoreContextClassLoader {
- sparkILoop.setContextClassLoader()
-
- var classLoader = Thread.currentThread().getContextClassLoader
- while (classLoader != null) {
- if (classLoader.getClass.getCanonicalName ==
- "org.apache.spark.util.MutableURLClassLoader") {
- val extraJarPath = classLoader.asInstanceOf[URLClassLoader].getURLs()
- // Check if the file exists. Otherwise an exception will be thrown.
- .filter { u => u.getProtocol == "file" && new File(u.getPath).isFile }
- // Livy rsc and repl are also in the extra jars list. Filter them out.
- .filterNot { u => Paths.get(u.toURI).getFileName.toString.startsWith("livy-") }
- // Some bad spark packages depend on the wrong version of scala-reflect. Blacklist it.
- .filterNot { u =>
- Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect")
- }
-
- extraJarPath.foreach { p => debug(s"Adding $p to Scala interpreter's class path...") }
- sparkILoop.addUrlsToClassPath(extraJarPath: _*)
- classLoader = null
- } else {
- classLoader = classLoader.getParent
- }
- }
-
- createSparkContext(conf)
- }
-
- sparkContext
- }
-
- override def close(): Unit = synchronized {
- if (sparkContext != null) {
- sparkContext.stop()
- sparkContext = null
- }
-
- if (sparkILoop != null) {
- sparkILoop.closeInterpreter()
- sparkILoop = null
- }
-
- if (sparkHttpServer != null) {
- val method = sparkHttpServer.getClass.getMethod("stop")
- method.setAccessible(true)
- method.invoke(sparkHttpServer)
- sparkHttpServer = null
- }
- }
-
- override protected def isStarted(): Boolean = {
- sparkContext != null && sparkILoop != null
- }
-
- override protected def interpret(code: String): Result = {
- sparkILoop.interpret(code)
- }
-
- override protected def valueOfTerm(name: String): Option[Any] = {
- // IMain#valueOfTerm will always return None, so use other way instead.
- Option(sparkILoop.lastRequest.lineRep.call("$result"))
- }
-
- protected def bind(name: String, tpe: String, value: Object, modifier: List[String]): Unit = {
- sparkILoop.beQuietDuring {
- sparkILoop.bind(name, tpe, value, modifier)
- }
- }
-
- private def startHttpServer(outputDir: File): Option[(Object, String)] = {
- try {
- val httpServerClass = Class.forName("org.apache.spark.HttpServer")
- val securityManager = {
- val constructor = Class.forName("org.apache.spark.SecurityManager")
- .getConstructor(classOf[SparkConf])
- constructor.setAccessible(true)
- constructor.newInstance(conf).asInstanceOf[Object]
- }
- val httpServerConstructor = httpServerClass
- .getConstructor(classOf[SparkConf],
- classOf[File],
- Class.forName("org.apache.spark.SecurityManager"),
- classOf[Int],
- classOf[String])
- httpServerConstructor.setAccessible(true)
- // Create Http Server
- val port = conf.getInt("spark.replClassServer.port", 0)
- val server = httpServerConstructor
- .newInstance(conf, outputDir, securityManager, new Integer(port), "HTTP server")
- .asInstanceOf[Object]
-
- // Start Http Server
- val startMethod = server.getClass.getMethod("start")
- startMethod.setAccessible(true)
- startMethod.invoke(server)
-
- // Get uri of this Http Server
- val uriMethod = server.getClass.getMethod("uri")
- uriMethod.setAccessible(true)
- val uri = uriMethod.invoke(server).asInstanceOf[String]
- Some((server, uri))
- } catch {
- // Spark 2.0+ removed HttpServer, so return null instead.
- case NonFatal(e) =>
- None
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/scala-2.11/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.11/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala b/repl/scala-2.11/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
new file mode 100644
index 0000000..94cd241
--- /dev/null
+++ b/repl/scala-2.11/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import java.io.File
+import java.net.URLClassLoader
+import java.nio.file.{Files, Paths}
+
+import scala.tools.nsc.Settings
+import scala.tools.nsc.interpreter.JPrintWriter
+import scala.tools.nsc.interpreter.Results.Result
+import scala.util.control.NonFatal
+
+import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.repl.SparkILoop
+
+/**
+ * Scala 2.11 version of SparkInterpreter
+ */
+class SparkInterpreter(conf: SparkConf)
+ extends AbstractSparkInterpreter with SparkContextInitializer {
+
+ protected var sparkContext: SparkContext = _
+ private var sparkILoop: SparkILoop = _
+ private var sparkHttpServer: Object = _
+
+ override def start(): SparkContext = {
+ require(sparkILoop == null)
+
+ val rootDir = conf.get("spark.repl.classdir", System.getProperty("java.io.tmpdir"))
+ val outputDir = Files.createTempDirectory(Paths.get(rootDir), "spark").toFile
+ outputDir.deleteOnExit()
+ conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath)
+
+ // Only Spark1 requires to create http server, Spark2 removes HttpServer class.
+ startHttpServer(outputDir).foreach { case (server, uri) =>
+ sparkHttpServer = server
+ conf.set("spark.repl.class.uri", uri)
+ }
+
+ val settings = new Settings()
+ settings.processArguments(List("-Yrepl-class-based",
+ "-Yrepl-outdir", s"${outputDir.getAbsolutePath}"), true)
+ settings.usejavacp.value = true
+ settings.embeddedDefaults(Thread.currentThread().getContextClassLoader())
+
+ sparkILoop = new SparkILoop(None, new JPrintWriter(outputStream, true))
+ sparkILoop.settings = settings
+ sparkILoop.createInterpreter()
+ sparkILoop.initializeSynchronous()
+
+ restoreContextClassLoader {
+ sparkILoop.setContextClassLoader()
+
+ var classLoader = Thread.currentThread().getContextClassLoader
+ while (classLoader != null) {
+ if (classLoader.getClass.getCanonicalName ==
+ "org.apache.spark.util.MutableURLClassLoader") {
+ val extraJarPath = classLoader.asInstanceOf[URLClassLoader].getURLs()
+ // Check if the file exists. Otherwise an exception will be thrown.
+ .filter { u => u.getProtocol == "file" && new File(u.getPath).isFile }
+ // Livy rsc and repl are also in the extra jars list. Filter them out.
+ .filterNot { u => Paths.get(u.toURI).getFileName.toString.startsWith("livy-") }
+ // Some bad spark packages depend on the wrong version of scala-reflect. Blacklist it.
+ .filterNot { u =>
+ Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect")
+ }
+
+ extraJarPath.foreach { p => debug(s"Adding $p to Scala interpreter's class path...") }
+ sparkILoop.addUrlsToClassPath(extraJarPath: _*)
+ classLoader = null
+ } else {
+ classLoader = classLoader.getParent
+ }
+ }
+
+ createSparkContext(conf)
+ }
+
+ sparkContext
+ }
+
+ override def close(): Unit = synchronized {
+ if (sparkContext != null) {
+ sparkContext.stop()
+ sparkContext = null
+ }
+
+ if (sparkILoop != null) {
+ sparkILoop.closeInterpreter()
+ sparkILoop = null
+ }
+
+ if (sparkHttpServer != null) {
+ val method = sparkHttpServer.getClass.getMethod("stop")
+ method.setAccessible(true)
+ method.invoke(sparkHttpServer)
+ sparkHttpServer = null
+ }
+ }
+
+ override protected def isStarted(): Boolean = {
+ sparkContext != null && sparkILoop != null
+ }
+
+ override protected def interpret(code: String): Result = {
+ sparkILoop.interpret(code)
+ }
+
+ override protected def valueOfTerm(name: String): Option[Any] = {
+ // IMain#valueOfTerm will always return None, so use other way instead.
+ Option(sparkILoop.lastRequest.lineRep.call("$result"))
+ }
+
+ protected def bind(name: String, tpe: String, value: Object, modifier: List[String]): Unit = {
+ sparkILoop.beQuietDuring {
+ sparkILoop.bind(name, tpe, value, modifier)
+ }
+ }
+
+ private def startHttpServer(outputDir: File): Option[(Object, String)] = {
+ try {
+ val httpServerClass = Class.forName("org.apache.spark.HttpServer")
+ val securityManager = {
+ val constructor = Class.forName("org.apache.spark.SecurityManager")
+ .getConstructor(classOf[SparkConf])
+ constructor.setAccessible(true)
+ constructor.newInstance(conf).asInstanceOf[Object]
+ }
+ val httpServerConstructor = httpServerClass
+ .getConstructor(classOf[SparkConf],
+ classOf[File],
+ Class.forName("org.apache.spark.SecurityManager"),
+ classOf[Int],
+ classOf[String])
+ httpServerConstructor.setAccessible(true)
+ // Create Http Server
+ val port = conf.getInt("spark.replClassServer.port", 0)
+ val server = httpServerConstructor
+ .newInstance(conf, outputDir, securityManager, new Integer(port), "HTTP server")
+ .asInstanceOf[Object]
+
+ // Start Http Server
+ val startMethod = server.getClass.getMethod("start")
+ startMethod.setAccessible(true)
+ startMethod.invoke(server)
+
+ // Get uri of this Http Server
+ val uriMethod = server.getClass.getMethod("uri")
+ uriMethod.setAccessible(true)
+ val uri = uriMethod.invoke(server).asInstanceOf[String]
+ Some((server, uri))
+ } catch {
+ // Spark 2.0+ removed HttpServer, so return null instead.
+ case NonFatal(e) =>
+ None
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/scala-2.11/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.11/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala b/repl/scala-2.11/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
deleted file mode 100644
index a2496a1..0000000
--- a/repl/scala-2.11/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import org.scalatest._
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-
-class SparkInterpreterSpec extends FunSpec with Matchers with LivyBaseUnitTestSuite {
- describe("SparkInterpreter") {
- val interpreter = new SparkInterpreter(null)
-
- it("should parse Scala compile error.") {
- // Regression test for LIVY-.
- val error =
- """<console>:27: error: type mismatch;
- | found : Int
- | required: String
- | sc.setJobGroup(groupName, groupName, true)
- | ^
- |<console>:27: error: type mismatch;
- | found : Int
- | required: String
- | sc.setJobGroup(groupName, groupName, true)
- | ^
- |""".stripMargin
-
- val parsedError = AbstractSparkInterpreter.KEEP_NEWLINE_REGEX.split(error)
-
- val expectedTraceback = parsedError.tail
-
- val (ename, traceback) = interpreter.parseError(error)
- ename shouldBe "<console>:27: error: type mismatch;"
- traceback shouldBe expectedTraceback
- }
-
- it("should parse Scala runtime error.") {
- val error =
- """java.lang.RuntimeException: message
- | ... 48 elided
- |
- |Tailing message""".stripMargin
-
- val parsedError = AbstractSparkInterpreter.KEEP_NEWLINE_REGEX.split(error)
-
- val expectedTraceback = parsedError.tail
-
- val (ename, traceback) = interpreter.parseError(error)
- ename shouldBe "java.lang.RuntimeException: message"
- traceback shouldBe expectedTraceback
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/scala-2.11/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.11/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala b/repl/scala-2.11/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala
new file mode 100644
index 0000000..d922034
--- /dev/null
+++ b/repl/scala-2.11/src/test/scala/org/apache/livy/repl/SparkInterpreterSpec.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import org.scalatest._
+
+import org.apache.livy.LivyBaseUnitTestSuite
+
+class SparkInterpreterSpec extends FunSpec with Matchers with LivyBaseUnitTestSuite {
+ describe("SparkInterpreter") {
+ val interpreter = new SparkInterpreter(null)
+
+ it("should parse Scala compile error.") {
+ // Regression test for LIVY-.
+ val error =
+ """<console>:27: error: type mismatch;
+ | found : Int
+ | required: String
+ | sc.setJobGroup(groupName, groupName, true)
+ | ^
+ |<console>:27: error: type mismatch;
+ | found : Int
+ | required: String
+ | sc.setJobGroup(groupName, groupName, true)
+ | ^
+ |""".stripMargin
+
+ val parsedError = AbstractSparkInterpreter.KEEP_NEWLINE_REGEX.split(error)
+
+ val expectedTraceback = parsedError.tail
+
+ val (ename, traceback) = interpreter.parseError(error)
+ ename shouldBe "<console>:27: error: type mismatch;"
+ traceback shouldBe expectedTraceback
+ }
+
+ it("should parse Scala runtime error.") {
+ val error =
+ """java.lang.RuntimeException: message
+ | ... 48 elided
+ |
+ |Tailing message""".stripMargin
+
+ val parsedError = AbstractSparkInterpreter.KEEP_NEWLINE_REGEX.split(error)
+
+ val expectedTraceback = parsedError.tail
+
+ val (ename, traceback) = interpreter.parseError(error)
+ ename shouldBe "java.lang.RuntimeException: message"
+ traceback shouldBe expectedTraceback
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/resources/fake_shell.py
----------------------------------------------------------------------
diff --git a/repl/src/main/resources/fake_shell.py b/repl/src/main/resources/fake_shell.py
index 1a8e24e..534e0df 100644
--- a/repl/src/main/resources/fake_shell.py
+++ b/repl/src/main/resources/fake_shell.py
@@ -174,7 +174,7 @@ class PySparkJobProcessorImpl(object):
return os.path.join(job_context.get_local_tmp_dir_path(), '__livy__')
class Scala:
- extends = ['com.cloudera.livy.repl.PySparkJobProcessor']
+ extends = ['org.apache.livy.repl.PySparkJobProcessor']
class ExecutionError(Exception):
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/AbstractSparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/AbstractSparkInterpreter.scala b/repl/src/main/scala/com/cloudera/livy/repl/AbstractSparkInterpreter.scala
deleted file mode 100644
index cffa69d..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/AbstractSparkInterpreter.scala
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import java.io.ByteArrayOutputStream
-
-import scala.tools.nsc.interpreter.Results
-
-import org.apache.spark.rdd.RDD
-import org.json4s.DefaultFormats
-import org.json4s.Extraction
-import org.json4s.JsonAST._
-import org.json4s.JsonDSL._
-
-import com.cloudera.livy.Logging
-
-object AbstractSparkInterpreter {
- private[repl] val KEEP_NEWLINE_REGEX = """(?<=\n)""".r
- private val MAGIC_REGEX = "^%(\\w+)\\W*(.*)".r
-}
-
-abstract class AbstractSparkInterpreter extends Interpreter with Logging {
- import AbstractSparkInterpreter._
-
- private implicit def formats = DefaultFormats
-
- protected val outputStream = new ByteArrayOutputStream()
-
- final def kind: String = "spark"
-
- protected def isStarted(): Boolean
-
- protected def interpret(code: String): Results.Result
-
- protected def valueOfTerm(name: String): Option[Any]
-
- override protected[repl] def execute(code: String): Interpreter.ExecuteResponse =
- restoreContextClassLoader {
- require(isStarted())
-
- executeLines(code.trim.split("\n").toList, Interpreter.ExecuteSuccess(JObject(
- (TEXT_PLAIN, JString(""))
- )))
- }
-
- private def executeMagic(magic: String, rest: String): Interpreter.ExecuteResponse = {
- magic match {
- case "json" => executeJsonMagic(rest)
- case "table" => executeTableMagic(rest)
- case _ =>
- Interpreter.ExecuteError("UnknownMagic", f"Unknown magic command $magic")
- }
- }
-
- private def executeJsonMagic(name: String): Interpreter.ExecuteResponse = {
- try {
- val value = valueOfTerm(name) match {
- case Some(obj: RDD[_]) => obj.asInstanceOf[RDD[_]].take(10)
- case Some(obj) => obj
- case None => return Interpreter.ExecuteError("NameError", f"Value $name does not exist")
- }
-
- Interpreter.ExecuteSuccess(JObject(
- (APPLICATION_JSON, Extraction.decompose(value))
- ))
- } catch {
- case _: Throwable =>
- Interpreter.ExecuteError("ValueError", "Failed to convert value into a JSON value")
- }
- }
-
- private class TypesDoNotMatch extends Exception
-
- private def convertTableType(value: JValue): String = {
- value match {
- case (JNothing | JNull) => "NULL_TYPE"
- case JBool(_) => "BOOLEAN_TYPE"
- case JString(_) => "STRING_TYPE"
- case JInt(_) => "BIGINT_TYPE"
- case JDouble(_) => "DOUBLE_TYPE"
- case JDecimal(_) => "DECIMAL_TYPE"
- case JArray(arr) =>
- if (allSameType(arr.iterator)) {
- "ARRAY_TYPE"
- } else {
- throw new TypesDoNotMatch
- }
- case JObject(obj) =>
- if (allSameType(obj.iterator.map(_._2))) {
- "MAP_TYPE"
- } else {
- throw new TypesDoNotMatch
- }
- }
- }
-
- private def allSameType(values: Iterator[JValue]): Boolean = {
- if (values.hasNext) {
- val type_name = convertTableType(values.next())
- values.forall { case value => type_name.equals(convertTableType(value)) }
- } else {
- true
- }
- }
-
- private def executeTableMagic(name: String): Interpreter.ExecuteResponse = {
- val value = valueOfTerm(name) match {
- case Some(obj: RDD[_]) => obj.asInstanceOf[RDD[_]].take(10)
- case Some(obj) => obj
- case None => return Interpreter.ExecuteError("NameError", f"Value $name does not exist")
- }
-
- extractTableFromJValue(Extraction.decompose(value))
- }
-
- private def extractTableFromJValue(value: JValue): Interpreter.ExecuteResponse = {
- // Convert the value into JSON and map it to a table.
- val rows: List[JValue] = value match {
- case JArray(arr) => arr
- case _ => List(value)
- }
-
- try {
- val headers = scala.collection.mutable.Map[String, Map[String, String]]()
-
- val data = rows.map { case row =>
- val cols: List[JField] = row match {
- case JArray(arr: List[JValue]) =>
- arr.zipWithIndex.map { case (v, index) => JField(index.toString, v) }
- case JObject(obj) => obj.sortBy(_._1)
- case value: JValue => List(JField("0", value))
- }
-
- cols.map { case (k, v) =>
- val typeName = convertTableType(v)
-
- headers.get(k) match {
- case Some(header) =>
- if (header.get("type").get != typeName) {
- throw new TypesDoNotMatch
- }
- case None =>
- headers.put(k, Map(
- "type" -> typeName,
- "name" -> k
- ))
- }
-
- v
- }
- }
-
- Interpreter.ExecuteSuccess(
- APPLICATION_LIVY_TABLE_JSON -> (
- ("headers" -> headers.toSeq.sortBy(_._1).map(_._2)) ~ ("data" -> data)
- ))
- } catch {
- case _: TypesDoNotMatch =>
- Interpreter.ExecuteError("TypeError", "table rows have different types")
- }
- }
-
- private def executeLines(
- lines: List[String],
- resultFromLastLine: Interpreter.ExecuteResponse): Interpreter.ExecuteResponse = {
- lines match {
- case Nil => resultFromLastLine
- case head :: tail =>
- val result = executeLine(head)
-
- result match {
- case Interpreter.ExecuteIncomplete() =>
- tail match {
- case Nil =>
- // ExecuteIncomplete could be caused by an actual incomplete statements (e.g. "sc.")
- // or statements with just comments.
- // To distinguish them, reissue the same statement wrapped in { }.
- // If it is an actual incomplete statement, the interpreter will return an error.
- // If it is some comment, the interpreter will return success.
- executeLine(s"{\n$head\n}") match {
- case Interpreter.ExecuteIncomplete() | Interpreter.ExecuteError(_, _, _) =>
- // Return the original error so users won't get confusing error message.
- result
- case _ => resultFromLastLine
- }
- case next :: nextTail =>
- executeLines(head + "\n" + next :: nextTail, resultFromLastLine)
- }
- case Interpreter.ExecuteError(_, _, _) =>
- result
-
- case _ =>
- executeLines(tail, result)
- }
- }
- }
-
- private def executeLine(code: String): Interpreter.ExecuteResponse = {
- code match {
- case MAGIC_REGEX(magic, rest) =>
- executeMagic(magic, rest)
- case _ =>
- scala.Console.withOut(outputStream) {
- interpret(code) match {
- case Results.Success =>
- Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> readStdout()
- )
- case Results.Incomplete => Interpreter.ExecuteIncomplete()
- case Results.Error =>
- val (ename, traceback) = parseError(readStdout())
- Interpreter.ExecuteError("Error", ename, traceback)
- }
- }
- }
- }
-
- protected[repl] def parseError(stdout: String): (String, Seq[String]) = {
- // An example of Scala compile error message:
- // <console>:27: error: type mismatch;
- // found : Int
- // required: Boolean
-
- // An example of Scala runtime exception error message:
- // java.lang.RuntimeException: message
- // at .error(<console>:11)
- // ... 32 elided
-
- // Return the first line as ename. Lines following as traceback.
-
- val lines = KEEP_NEWLINE_REGEX.split(stdout)
- val ename = lines.headOption.map(_.trim).getOrElse("unknown error")
- val traceback = lines.tail
-
- (ename, traceback)
- }
-
- protected def restoreContextClassLoader[T](fn: => T): T = {
- val currentClassLoader = Thread.currentThread().getContextClassLoader()
- try {
- fn
- } finally {
- Thread.currentThread().setContextClassLoader(currentClassLoader)
- }
- }
-
- private def readStdout() = {
- val output = outputStream.toString("UTF-8").trim
- outputStream.reset()
-
- output
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/BypassPySparkJob.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/BypassPySparkJob.scala b/repl/src/main/scala/com/cloudera/livy/repl/BypassPySparkJob.scala
deleted file mode 100644
index 1550129..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/BypassPySparkJob.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.repl
-
-import java.nio.charset.StandardCharsets
-
-import com.cloudera.livy.{Job, JobContext}
-
-class BypassPySparkJob(
- serializedJob: Array[Byte],
- replDriver: ReplDriver) extends Job[Array[Byte]] {
-
- override def call(jc: JobContext): Array[Byte] = {
- val interpreter = replDriver.interpreter
- require(interpreter != null && interpreter.isInstanceOf[PythonInterpreter])
- val pi = interpreter.asInstanceOf[PythonInterpreter]
-
- val resultByteArray = pi.pysparkJobProcessor.processBypassJob(serializedJob)
- val resultString = new String(resultByteArray, StandardCharsets.UTF_8)
- if (resultString.startsWith("Client job error:")) {
- throw new PythonJobException(resultString)
- }
- resultByteArray
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/Interpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/Interpreter.scala b/repl/src/main/scala/com/cloudera/livy/repl/Interpreter.scala
deleted file mode 100644
index 5d56725..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/Interpreter.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import org.apache.spark.SparkContext
-import org.json4s.JObject
-
-object Interpreter {
- abstract class ExecuteResponse
-
- case class ExecuteSuccess(content: JObject) extends ExecuteResponse
- case class ExecuteError(ename: String,
- evalue: String,
- traceback: Seq[String] = Seq()) extends ExecuteResponse
- case class ExecuteIncomplete() extends ExecuteResponse
- case class ExecuteAborted(message: String) extends ExecuteResponse
-}
-
-trait Interpreter {
- import Interpreter._
-
- def kind: String
-
- /**
- * Start the Interpreter.
- *
- * @return A SparkContext
- */
- def start(): SparkContext
-
- /**
- * Execute the code and return the result, it may
- * take some time to execute.
- */
- protected[repl] def execute(code: String): ExecuteResponse
-
- /** Shut down the interpreter. */
- def close(): Unit
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala b/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala
deleted file mode 100644
index 924c576..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import java.io.{BufferedReader, InputStreamReader, IOException, PrintWriter}
-import java.util.concurrent.locks.ReentrantLock
-
-import scala.concurrent.Promise
-import scala.io.Source
-
-import org.apache.spark.SparkContext
-import org.json4s.JValue
-
-import com.cloudera.livy.{Logging, Utils}
-import com.cloudera.livy.client.common.ClientConf
-
-private sealed trait Request
-private case class ExecuteRequest(code: String, promise: Promise[JValue]) extends Request
-private case class ShutdownRequest(promise: Promise[Unit]) extends Request
-
-/**
- * Abstract class that describes an interpreter that is running in a separate process.
- *
- * This type is not thread safe, so must be protected by a mutex.
- *
- * @param process
- */
-abstract class ProcessInterpreter(process: Process)
- extends Interpreter with Logging {
- protected[this] val stdin = new PrintWriter(process.getOutputStream)
- protected[this] val stdout = new BufferedReader(new InputStreamReader(process.getInputStream), 1)
-
- override def start(): SparkContext = {
- waitUntilReady()
-
- if (ClientConf.TEST_MODE) {
- null.asInstanceOf[SparkContext]
- } else {
- SparkContext.getOrCreate()
- }
- }
-
- override protected[repl] def execute(code: String): Interpreter.ExecuteResponse = {
- try {
- sendExecuteRequest(code)
- } catch {
- case e: Throwable =>
- Interpreter.ExecuteError(e.getClass.getName, e.getMessage)
- }
- }
-
- override def close(): Unit = {
- if (Utils.isProcessAlive(process)) {
- logger.info("Shutting down process")
- sendShutdownRequest()
-
- try {
- process.getInputStream.close()
- process.getOutputStream.close()
- } catch {
- case _: IOException =>
- }
-
- try {
- process.destroy()
- } finally {
- logger.info("process has been shut down")
- }
- }
- }
-
- protected def sendExecuteRequest(request: String): Interpreter.ExecuteResponse
-
- protected def sendShutdownRequest(): Unit = {}
-
- protected def waitUntilReady(): Unit
-
- private[this] val stderrLock = new ReentrantLock()
- private[this] var stderrLines = Seq[String]()
-
- protected def takeErrorLines(): String = {
- stderrLock.lock()
- try {
- val lines = stderrLines
- stderrLines = Seq()
- lines.mkString("\n")
- } finally {
- stderrLock.unlock()
- }
- }
-
- private[this] val stderrThread = new Thread("process stderr thread") {
- override def run() = {
- val lines = Source.fromInputStream(process.getErrorStream).getLines()
-
- for (line <- lines) {
- stderrLock.lock()
- try {
- stderrLines :+= line
- } finally {
- stderrLock.unlock()
- }
- }
- }
- }
-
- stderrThread.setDaemon(true)
- stderrThread.start()
-
- private[this] val processWatcherThread = new Thread("process watcher thread") {
- override def run() = {
- val exitCode = process.waitFor()
- if (exitCode != 0) {
- error(f"Process has died with $exitCode")
- error(stderrLines.mkString("\n"))
- }
- }
- }
-
- processWatcherThread.setDaemon(true)
- processWatcherThread.start()
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/PySparkJobProcessor.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/PySparkJobProcessor.scala b/repl/src/main/scala/com/cloudera/livy/repl/PySparkJobProcessor.scala
deleted file mode 100644
index f0cdbd9..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/PySparkJobProcessor.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.repl
-
-trait PySparkJobProcessor {
- def processBypassJob(job: Array[Byte]): Array[Byte]
-
- def addFile(path: String)
-
- def addPyFile(path: String)
-
- def getLocalTmpDirPath: String
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/PythonInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/PythonInterpreter.scala b/repl/src/main/scala/com/cloudera/livy/repl/PythonInterpreter.scala
deleted file mode 100644
index c744c40..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/PythonInterpreter.scala
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import java.io._
-import java.lang.ProcessBuilder.Redirect
-import java.lang.reflect.Proxy
-import java.nio.file.{Files, Paths}
-
-import scala.annotation.tailrec
-import scala.collection.mutable.ArrayBuffer
-import scala.collection.JavaConverters._
-import scala.util.control.NonFatal
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.json4s.{DefaultFormats, JValue}
-import org.json4s.JsonAST.JObject
-import org.json4s.jackson.JsonMethods._
-import org.json4s.jackson.Serialization.write
-import py4j._
-import py4j.reflection.PythonProxyHandler
-
-import com.cloudera.livy.Logging
-import com.cloudera.livy.client.common.ClientConf
-import com.cloudera.livy.rsc.BaseProtocol
-import com.cloudera.livy.rsc.driver.BypassJobWrapper
-import com.cloudera.livy.sessions._
-
-// scalastyle:off println
-object PythonInterpreter extends Logging {
-
- def apply(conf: SparkConf, kind: Kind): Interpreter = {
- val pythonExec = kind match {
- case PySpark() => sys.env.getOrElse("PYSPARK_PYTHON", "python")
- case PySpark3() => sys.env.getOrElse("PYSPARK3_PYTHON", "python3")
- case _ => throw new IllegalArgumentException(s"Unknown kind: $kind")
- }
-
- val gatewayServer = new GatewayServer(null, 0)
- gatewayServer.start()
-
- val builder = new ProcessBuilder(Seq(pythonExec, createFakeShell().toString).asJava)
-
- val env = builder.environment()
-
- val pythonPath = sys.env.getOrElse("PYTHONPATH", "")
- .split(File.pathSeparator)
- .++(if (!ClientConf.TEST_MODE) findPySparkArchives() else Nil)
- .++(if (!ClientConf.TEST_MODE) findPyFiles() else Nil)
-
- env.put("PYSPARK_PYTHON", pythonExec)
- env.put("PYTHONPATH", pythonPath.mkString(File.pathSeparator))
- env.put("PYTHONUNBUFFERED", "YES")
- env.put("PYSPARK_GATEWAY_PORT", "" + gatewayServer.getListeningPort)
- env.put("SPARK_HOME", sys.env.getOrElse("SPARK_HOME", "."))
- env.put("LIVY_SPARK_MAJOR_VERSION", conf.get("spark.livy.spark_major_version", "1"))
- builder.redirectError(Redirect.PIPE)
- val process = builder.start()
- new PythonInterpreter(process, gatewayServer, kind.toString)
- }
-
- private def findPySparkArchives(): Seq[String] = {
- sys.env.get("PYSPARK_ARCHIVES_PATH")
- .map(_.split(",").toSeq)
- .getOrElse {
- sys.env.get("SPARK_HOME").map { sparkHome =>
- val pyLibPath = Seq(sparkHome, "python", "lib").mkString(File.separator)
- val pyArchivesFile = new File(pyLibPath, "pyspark.zip")
- require(pyArchivesFile.exists(),
- "pyspark.zip not found; cannot run pyspark application in YARN mode.")
-
- val py4jFile = Files.newDirectoryStream(Paths.get(pyLibPath), "py4j-*-src.zip")
- .iterator()
- .next()
- .toFile
-
- require(py4jFile.exists(),
- "py4j-*-src.zip not found; cannot run pyspark application in YARN mode.")
- Seq(pyArchivesFile.getAbsolutePath, py4jFile.getAbsolutePath)
- }.getOrElse(Seq())
- }
- }
-
- private def findPyFiles(): Seq[String] = {
- val pyFiles = sys.props.getOrElse("spark.submit.pyFiles", "").split(",")
-
- if (sys.env.getOrElse("SPARK_YARN_MODE", "") == "true") {
- // In spark mode, these files have been localized into the current directory.
- pyFiles.map { file =>
- val name = new File(file).getName
- new File(name).getAbsolutePath
- }
- } else {
- pyFiles
- }
- }
-
- private def createFakeShell(): File = {
- val source: InputStream = getClass.getClassLoader.getResourceAsStream("fake_shell.py")
-
- val file = Files.createTempFile("", "").toFile
- file.deleteOnExit()
-
- val sink = new FileOutputStream(file)
- val buf = new Array[Byte](1024)
- var n = source.read(buf)
-
- while (n > 0) {
- sink.write(buf, 0, n)
- n = source.read(buf)
- }
-
- source.close()
- sink.close()
-
- file
- }
-
- private def initiatePy4jCallbackGateway(server: GatewayServer): PySparkJobProcessor = {
- val f = server.getClass.getDeclaredField("gateway")
- f.setAccessible(true)
- val gateway = f.get(server).asInstanceOf[Gateway]
- val command: String = "f" + Protocol.ENTRY_POINT_OBJECT_ID + ";" +
- "com.cloudera.livy.repl.PySparkJobProcessor"
- getPythonProxy(command, gateway).asInstanceOf[PySparkJobProcessor]
- }
-
- // This method is a hack to get around the classLoader issues faced in py4j 0.8.2.1 for
- // dynamically adding jars to the driver. The change is to use the context classLoader instead
- // of the system classLoader when initiating a new Proxy instance
- // ISSUE - https://issues.apache.org/jira/browse/SPARK-6047
- // FIX - https://github.com/bartdag/py4j/pull/196
- private def getPythonProxy(commandPart: String, gateway: Gateway): Any = {
- val proxyString = commandPart.substring(1, commandPart.length)
- val parts = proxyString.split(";")
- val length: Int = parts.length
- val interfaces = ArrayBuffer.fill[Class[_]](length - 1){ null }
- if (length < 2) {
- throw new Py4JException("Invalid Python Proxy.")
- }
- else {
- var proxy: Int = 1
- while (proxy < length) {
- try {
- interfaces(proxy - 1) = Class.forName(parts(proxy))
- if (!interfaces(proxy - 1).isInterface) {
- throw new Py4JException("This class " + parts(proxy) +
- " is not an interface and cannot be used as a Python Proxy.")
- }
- } catch {
- case exception: ClassNotFoundException => {
- throw new Py4JException("Invalid interface name: " + parts(proxy))
- }
- }
- proxy += 1
- }
-
- val pythonProxyHandler = try {
- classOf[PythonProxyHandler].getConstructor(classOf[String], classOf[Gateway])
- .newInstance(parts(0), gateway)
- } catch {
- case NonFatal(e) =>
- classOf[PythonProxyHandler].getConstructor(classOf[String],
- Class.forName("py4j.CallbackClient"), classOf[Gateway])
- .newInstance(parts(0), gateway.getCallbackClient, gateway)
- }
-
- Proxy.newProxyInstance(Thread.currentThread.getContextClassLoader,
- interfaces.toArray, pythonProxyHandler.asInstanceOf[PythonProxyHandler])
- }
- }
-}
-
-private class PythonInterpreter(
- process: Process,
- gatewayServer: GatewayServer,
- pyKind: String)
- extends ProcessInterpreter(process)
- with Logging
-{
- implicit val formats = DefaultFormats
-
- override def kind: String = pyKind
-
- private[repl] val pysparkJobProcessor =
- PythonInterpreter.initiatePy4jCallbackGateway(gatewayServer)
-
- override def close(): Unit = {
- try {
- super.close()
- } finally {
- gatewayServer.shutdown()
- }
- }
-
- @tailrec
- final override protected def waitUntilReady(): Unit = {
- val READY_REGEX = "READY\\(port=([0-9]+)\\)".r
- stdout.readLine() match {
- case null =>
- case READY_REGEX(port) => updatePythonGatewayPort(port.toInt)
- case _ => waitUntilReady()
- }
- }
-
- override protected def sendExecuteRequest(code: String): Interpreter.ExecuteResponse = {
- sendRequest(Map("msg_type" -> "execute_request", "content" -> Map("code" -> code))) match {
- case Some(response) =>
- assert((response \ "msg_type").extract[String] == "execute_reply")
-
- val content = response \ "content"
-
- (content \ "status").extract[String] match {
- case "ok" =>
- Interpreter.ExecuteSuccess((content \ "data").extract[JObject])
- case "error" =>
- val ename = (content \ "ename").extract[String]
- val evalue = (content \ "evalue").extract[String]
- val traceback = (content \ "traceback").extract[Seq[String]]
-
- Interpreter.ExecuteError(ename, evalue, traceback)
- case status =>
- Interpreter.ExecuteError("Internal Error", f"Unknown status $status")
- }
- case None =>
- Interpreter.ExecuteAborted(takeErrorLines())
- }
- }
-
- override protected def sendShutdownRequest(): Unit = {
- sendRequest(Map(
- "msg_type" -> "shutdown_request",
- "content" -> ()
- )).foreach { case rep =>
- warn(f"process failed to shut down while returning $rep")
- }
- }
-
- private def sendRequest(request: Map[String, Any]): Option[JValue] = {
- stdin.println(write(request))
- stdin.flush()
-
- Option(stdout.readLine()).map { case line =>
- parse(line)
- }
- }
-
- def addFile(path: String): Unit = {
- pysparkJobProcessor.addFile(path)
- }
-
- def addPyFile(driver: ReplDriver, conf: SparkConf, path: String): Unit = {
- val localCopyDir = new File(pysparkJobProcessor.getLocalTmpDirPath)
- val localCopyFile = driver.copyFileToLocal(localCopyDir, path, SparkContext.getOrCreate(conf))
- pysparkJobProcessor.addPyFile(localCopyFile.getPath)
- if (path.endsWith(".jar")) {
- driver.addLocalFileToClassLoader(localCopyFile)
- }
- }
-
- private def updatePythonGatewayPort(port: Int): Unit = {
- // The python gateway port can be 0 only when LivyConf.TEST_MODE is true
- // Py4j 0.10 has different API signature for "getCallbackClient", use reflection to handle it.
- if (port != 0) {
- val callbackClient = gatewayServer.getClass
- .getMethod("getCallbackClient")
- .invoke(gatewayServer)
-
- val field = Class.forName("py4j.CallbackClient").getDeclaredField("port")
- field.setAccessible(true)
- field.setInt(callbackClient, port.toInt)
- }
- }
-}
-
-case class PythonJobException(message: String) extends Exception(message) {}
-
-// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/ReplDriver.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/ReplDriver.scala b/repl/src/main/scala/com/cloudera/livy/repl/ReplDriver.scala
deleted file mode 100644
index 716e814..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/ReplDriver.scala
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import scala.concurrent.Await
-import scala.concurrent.duration.Duration
-
-import io.netty.channel.ChannelHandlerContext
-import org.apache.spark.SparkConf
-import org.apache.spark.api.java.JavaSparkContext
-
-import com.cloudera.livy.Logging
-import com.cloudera.livy.rsc.{BaseProtocol, ReplJobResults, RSCConf}
-import com.cloudera.livy.rsc.BaseProtocol.ReplState
-import com.cloudera.livy.rsc.driver._
-import com.cloudera.livy.rsc.rpc.Rpc
-import com.cloudera.livy.sessions._
-
-class ReplDriver(conf: SparkConf, livyConf: RSCConf)
- extends RSCDriver(conf, livyConf)
- with Logging {
-
- private[repl] var session: Session = _
-
- private val kind = Kind(livyConf.get(RSCConf.Entry.SESSION_KIND))
-
- private[repl] var interpreter: Interpreter = _
-
- override protected def initializeContext(): JavaSparkContext = {
- interpreter = kind match {
- case PySpark() => PythonInterpreter(conf, PySpark())
- case PySpark3() =>
- PythonInterpreter(conf, PySpark3())
- case Spark() => new SparkInterpreter(conf)
- case SparkR() => SparkRInterpreter(conf)
- }
- session = new Session(livyConf, interpreter, { s => broadcast(new ReplState(s.toString)) })
-
- Option(Await.result(session.start(), Duration.Inf))
- .map(new JavaSparkContext(_))
- .orNull
- }
-
- override protected def shutdownContext(): Unit = {
- if (session != null) {
- try {
- session.close()
- } finally {
- super.shutdownContext()
- }
- }
- }
-
- def handle(ctx: ChannelHandlerContext, msg: BaseProtocol.ReplJobRequest): Int = {
- session.execute(msg.code)
- }
-
- def handle(ctx: ChannelHandlerContext, msg: BaseProtocol.CancelReplJobRequest): Unit = {
- session.cancel(msg.id)
- }
-
- /**
- * Return statement results. Results are sorted by statement id.
- */
- def handle(ctx: ChannelHandlerContext, msg: BaseProtocol.GetReplJobResults): ReplJobResults = {
- val statements = if (msg.allResults) {
- session.statements.values.toArray
- } else {
- assert(msg.from != null)
- assert(msg.size != null)
- if (msg.size == 1) {
- session.statements.get(msg.from).toArray
- } else {
- val until = msg.from + msg.size
- session.statements.filterKeys(id => id >= msg.from && id < until).values.toArray
- }
- }
-
- // Update progress of statements when queried
- statements.foreach { s =>
- s.updateProgress(session.progressOfStatement(s.id))
- }
-
- new ReplJobResults(statements.sortBy(_.id))
- }
-
- override protected def createWrapper(msg: BaseProtocol.BypassJobRequest): BypassJobWrapper = {
- kind match {
- case PySpark() | PySpark3() => new BypassJobWrapper(this, msg.id,
- new BypassPySparkJob(msg.serializedJob, this))
- case _ => super.createWrapper(msg)
- }
- }
-
- override protected def addFile(path: String): Unit = {
- require(interpreter != null)
- interpreter match {
- case pi: PythonInterpreter => pi.addFile(path)
- case _ => super.addFile(path)
- }
- }
-
- override protected def addJarOrPyFile(path: String): Unit = {
- require(interpreter != null)
- interpreter match {
- case pi: PythonInterpreter => pi.addPyFile(this, conf, path)
- case _ => super.addJarOrPyFile(path)
- }
- }
-
- override protected def onClientAuthenticated(client: Rpc): Unit = {
- if (session != null) {
- client.call(new ReplState(session.state.toString))
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/Session.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/Session.scala b/repl/src/main/scala/com/cloudera/livy/repl/Session.scala
deleted file mode 100644
index 4f17978..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/Session.scala
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import java.util.{LinkedHashMap => JLinkedHashMap}
-import java.util.Map.Entry
-import java.util.concurrent.Executors
-import java.util.concurrent.atomic.AtomicInteger
-
-import scala.collection.JavaConverters._
-import scala.concurrent.{ExecutionContext, Future}
-import scala.concurrent.duration._
-
-import org.apache.spark.SparkContext
-import org.json4s.jackson.JsonMethods.{compact, render}
-import org.json4s.DefaultFormats
-import org.json4s.JsonDSL._
-
-import com.cloudera.livy.Logging
-import com.cloudera.livy.rsc.RSCConf
-import com.cloudera.livy.rsc.driver.{Statement, StatementState}
-import com.cloudera.livy.sessions._
-
-object Session {
- val STATUS = "status"
- val OK = "ok"
- val ERROR = "error"
- val EXECUTION_COUNT = "execution_count"
- val DATA = "data"
- val ENAME = "ename"
- val EVALUE = "evalue"
- val TRACEBACK = "traceback"
-}
-
-class Session(
- livyConf: RSCConf,
- interpreter: Interpreter,
- stateChangedCallback: SessionState => Unit = { _ => })
- extends Logging {
- import Session._
-
- private val interpreterExecutor = ExecutionContext.fromExecutorService(
- Executors.newSingleThreadExecutor())
-
- private val cancelExecutor = ExecutionContext.fromExecutorService(
- Executors.newSingleThreadExecutor())
-
- private implicit val formats = DefaultFormats
-
- @volatile private[repl] var _sc: Option[SparkContext] = None
-
- private var _state: SessionState = SessionState.NotStarted()
-
- // Number of statements kept in driver's memory
- private val numRetainedStatements = livyConf.getInt(RSCConf.Entry.RETAINED_STATEMENT_NUMBER)
-
- private val _statements = new JLinkedHashMap[Int, Statement] {
- protected override def removeEldestEntry(eldest: Entry[Int, Statement]): Boolean = {
- size() > numRetainedStatements
- }
- }.asScala
-
- private val newStatementId = new AtomicInteger(0)
-
- stateChangedCallback(_state)
-
- def start(): Future[SparkContext] = {
- val future = Future {
- changeState(SessionState.Starting())
- val sc = interpreter.start()
- _sc = Option(sc)
- changeState(SessionState.Idle())
- sc
- }(interpreterExecutor)
-
- future.onFailure { case _ => changeState(SessionState.Error()) }(interpreterExecutor)
- future
- }
-
- def kind: String = interpreter.kind
-
- def state: SessionState = _state
-
- def statements: collection.Map[Int, Statement] = _statements.synchronized {
- _statements.toMap
- }
-
- def execute(code: String): Int = {
- val statementId = newStatementId.getAndIncrement()
- val statement = new Statement(statementId, code, StatementState.Waiting, null)
- _statements.synchronized { _statements(statementId) = statement }
-
- Future {
- setJobGroup(statementId)
- statement.compareAndTransit(StatementState.Waiting, StatementState.Running)
-
- if (statement.state.get() == StatementState.Running) {
- statement.output = executeCode(statementId, code)
- }
-
- statement.compareAndTransit(StatementState.Running, StatementState.Available)
- statement.compareAndTransit(StatementState.Cancelling, StatementState.Cancelled)
- statement.updateProgress(1.0)
- }(interpreterExecutor)
-
- statementId
- }
-
- def cancel(statementId: Int): Unit = {
- val statementOpt = _statements.synchronized { _statements.get(statementId) }
- if (statementOpt.isEmpty) {
- return
- }
-
- val statement = statementOpt.get
- if (statement.state.get().isOneOf(
- StatementState.Available, StatementState.Cancelled, StatementState.Cancelling)) {
- return
- } else {
- // statement 1 is running and statement 2 is waiting. User cancels
- // statement 2 then cancels statement 1. The 2nd cancel call will loop and block the 1st
- // cancel call since cancelExecutor is single threaded. To avoid this, set the statement
- // state to cancelled when cancelling a waiting statement.
- statement.compareAndTransit(StatementState.Waiting, StatementState.Cancelled)
- statement.compareAndTransit(StatementState.Running, StatementState.Cancelling)
- }
-
- info(s"Cancelling statement $statementId...")
-
- Future {
- val deadline = livyConf.getTimeAsMs(RSCConf.Entry.JOB_CANCEL_TIMEOUT).millis.fromNow
-
- while (statement.state.get() == StatementState.Cancelling) {
- if (deadline.isOverdue()) {
- info(s"Failed to cancel statement $statementId.")
- statement.compareAndTransit(StatementState.Cancelling, StatementState.Cancelled)
- } else {
- _sc.foreach(_.cancelJobGroup(statementId.toString))
- if (statement.state.get() == StatementState.Cancelling) {
- Thread.sleep(livyConf.getTimeAsMs(RSCConf.Entry.JOB_CANCEL_TRIGGER_INTERVAL))
- }
- }
- }
-
- if (statement.state.get() == StatementState.Cancelled) {
- info(s"Statement $statementId cancelled.")
- }
- }(cancelExecutor)
- }
-
- def close(): Unit = {
- interpreterExecutor.shutdown()
- cancelExecutor.shutdown()
- interpreter.close()
- }
-
- /**
- * Get the current progress of given statement id.
- */
- def progressOfStatement(stmtId: Int): Double = {
- val jobGroup = statementIdToJobGroup(stmtId)
-
- _sc.map { sc =>
- val jobIds = sc.statusTracker.getJobIdsForGroup(jobGroup)
- val jobs = jobIds.flatMap { id => sc.statusTracker.getJobInfo(id) }
- val stages = jobs.flatMap { job =>
- job.stageIds().flatMap(sc.statusTracker.getStageInfo)
- }
-
- val taskCount = stages.map(_.numTasks).sum
- val completedTaskCount = stages.map(_.numCompletedTasks).sum
- if (taskCount == 0) {
- 0.0
- } else {
- completedTaskCount.toDouble / taskCount
- }
- }.getOrElse(0.0)
- }
-
- private def changeState(newState: SessionState): Unit = {
- synchronized {
- _state = newState
- }
- stateChangedCallback(newState)
- }
-
- private def executeCode(executionCount: Int, code: String): String = {
- changeState(SessionState.Busy())
-
- def transitToIdle() = {
- val executingLastStatement = executionCount == newStatementId.intValue() - 1
- if (_statements.isEmpty || executingLastStatement) {
- changeState(SessionState.Idle())
- }
- }
-
- val resultInJson = try {
- interpreter.execute(code) match {
- case Interpreter.ExecuteSuccess(data) =>
- transitToIdle()
-
- (STATUS -> OK) ~
- (EXECUTION_COUNT -> executionCount) ~
- (DATA -> data)
-
- case Interpreter.ExecuteIncomplete() =>
- transitToIdle()
-
- (STATUS -> ERROR) ~
- (EXECUTION_COUNT -> executionCount) ~
- (ENAME -> "Error") ~
- (EVALUE -> "incomplete statement") ~
- (TRACEBACK -> Seq.empty[String])
-
- case Interpreter.ExecuteError(ename, evalue, traceback) =>
- transitToIdle()
-
- (STATUS -> ERROR) ~
- (EXECUTION_COUNT -> executionCount) ~
- (ENAME -> ename) ~
- (EVALUE -> evalue) ~
- (TRACEBACK -> traceback)
-
- case Interpreter.ExecuteAborted(message) =>
- changeState(SessionState.Error())
-
- (STATUS -> ERROR) ~
- (EXECUTION_COUNT -> executionCount) ~
- (ENAME -> "Error") ~
- (EVALUE -> f"Interpreter died:\n$message") ~
- (TRACEBACK -> Seq.empty[String])
- }
- } catch {
- case e: Throwable =>
- error("Exception when executing code", e)
-
- transitToIdle()
-
- (STATUS -> ERROR) ~
- (EXECUTION_COUNT -> executionCount) ~
- (ENAME -> f"Internal Error: ${e.getClass.getName}") ~
- (EVALUE -> e.getMessage) ~
- (TRACEBACK -> Seq.empty[String])
- }
-
- compact(render(resultInJson))
- }
-
- private def setJobGroup(statementId: Int): String = {
- val jobGroup = statementIdToJobGroup(statementId)
- val cmd = Kind(interpreter.kind) match {
- case Spark() =>
- // A dummy value to avoid automatic value binding in scala REPL.
- s"""val _livyJobGroup$jobGroup = sc.setJobGroup("$jobGroup",""" +
- s""""Job group for statement $jobGroup")"""
- case PySpark() | PySpark3() =>
- s"""sc.setJobGroup("$jobGroup", "Job group for statement $jobGroup")"""
- case SparkR() =>
- interpreter.asInstanceOf[SparkRInterpreter].sparkMajorVersion match {
- case "1" =>
- s"""setJobGroup(sc, "$jobGroup", "Job group for statement $jobGroup", """ +
- "FALSE)"
- case "2" =>
- s"""setJobGroup("$jobGroup", "Job group for statement $jobGroup", FALSE)"""
- }
- }
- // Set the job group
- executeCode(statementId, cmd)
- }
-
- private def statementIdToJobGroup(statementId: Int): String = {
- statementId.toString
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala b/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala
deleted file mode 100644
index ba334da..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import org.apache.spark.{SparkConf, SparkContext}
-
-import com.cloudera.livy.Logging
-
-/**
- * A mixin trait for Spark entry point creation. This trait exists two different code path
- * separately for Spark1 and Spark2, depends on whether SparkSession exists or not.
- */
-trait SparkContextInitializer extends Logging {
- self: SparkInterpreter =>
-
- def createSparkContext(conf: SparkConf): Unit = {
- if (isSparkSessionPresent()) {
- spark2CreateContext(conf)
- } else {
- spark1CreateContext(conf)
- }
- }
-
- private def spark1CreateContext(conf: SparkConf): Unit = {
- sparkContext = SparkContext.getOrCreate(conf)
- var sqlContext: Object = null
-
- if (conf.getBoolean("spark.repl.enableHiveContext", false)) {
- try {
- val loader = Option(Thread.currentThread().getContextClassLoader)
- .getOrElse(getClass.getClassLoader)
- if (loader.getResource("hive-site.xml") == null) {
- warn("livy.repl.enable-hive-context is true but no hive-site.xml found on classpath.")
- }
-
- sqlContext = Class.forName("org.apache.spark.sql.hive.HiveContext")
- .getConstructor(classOf[SparkContext]).newInstance(sparkContext).asInstanceOf[Object]
- info("Created sql context (with Hive support).")
- } catch {
- case _: NoClassDefFoundError =>
- sqlContext = Class.forName("org.apache.spark.sql.SQLContext")
- .getConstructor(classOf[SparkContext]).newInstance(sparkContext).asInstanceOf[Object]
- info("Created sql context.")
- }
- } else {
- sqlContext = Class.forName("org.apache.spark.sql.SQLContext")
- .getConstructor(classOf[SparkContext]).newInstance(sparkContext).asInstanceOf[Object]
- info("Created sql context.")
- }
-
- bind("sc", "org.apache.spark.SparkContext", sparkContext, List("""@transient"""))
- bind("sqlContext", sqlContext.getClass.getCanonicalName, sqlContext, List("""@transient"""))
-
- execute("import org.apache.spark.SparkContext._")
- execute("import sqlContext.implicits._")
- execute("import sqlContext.sql")
- execute("import org.apache.spark.sql.functions._")
- }
-
- private def spark2CreateContext(conf: SparkConf): Unit = {
- val sparkClz = Class.forName("org.apache.spark.sql.SparkSession$")
- val sparkObj = sparkClz.getField("MODULE$").get(null)
-
- val builderMethod = sparkClz.getMethod("builder")
- val builder = builderMethod.invoke(sparkObj)
- builder.getClass.getMethod("config", classOf[SparkConf]).invoke(builder, conf)
-
- var spark: Object = null
- if (conf.get("spark.sql.catalogImplementation", "in-memory").toLowerCase == "hive") {
- if (sparkClz.getMethod("hiveClassesArePresent").invoke(sparkObj).asInstanceOf[Boolean]) {
- val loader = Option(Thread.currentThread().getContextClassLoader)
- .getOrElse(getClass.getClassLoader)
- if (loader.getResource("hive-site.xml") == null) {
- warn("livy.repl.enable-hive-context is true but no hive-site.xml found on classpath.")
- }
-
- builder.getClass.getMethod("enableHiveSupport").invoke(builder)
- spark = builder.getClass.getMethod("getOrCreate").invoke(builder)
- info("Created Spark session (with Hive support).")
- } else {
- spark = builder.getClass.getMethod("getOrCreate").invoke(builder)
- info("Created Spark session.")
- }
- } else {
- spark = builder.getClass.getMethod("getOrCreate").invoke(builder)
- info("Created Spark session.")
- }
-
- sparkContext = spark.getClass.getMethod("sparkContext").invoke(spark)
- .asInstanceOf[SparkContext]
-
- bind("spark", spark.getClass.getCanonicalName, spark, List("""@transient"""))
- bind("sc", "org.apache.spark.SparkContext", sparkContext, List("""@transient"""))
-
- execute("import org.apache.spark.SparkContext._")
- execute("import spark.implicits._")
- execute("import spark.sql")
- execute("import org.apache.spark.sql.functions._")
- }
-
- private def isSparkSessionPresent(): Boolean = {
- try {
- Class.forName("org.apache.spark.sql.SparkSession")
- true
- } catch {
- case _: ClassNotFoundException | _: NoClassDefFoundError => false
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala b/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
deleted file mode 100644
index 42f8e17..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import java.io.{File, FileOutputStream}
-import java.lang.ProcessBuilder.Redirect
-import java.nio.file.Files
-import java.util.concurrent.{CountDownLatch, Semaphore, TimeUnit}
-
-import scala.annotation.tailrec
-import scala.collection.JavaConverters._
-import scala.reflect.runtime.universe
-
-import org.apache.commons.codec.binary.Base64
-import org.apache.commons.lang.StringEscapeUtils
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.util.{ChildFirstURLClassLoader, MutableURLClassLoader, Utils}
-import org.json4s._
-import org.json4s.JsonDSL._
-
-import com.cloudera.livy.client.common.ClientConf
-import com.cloudera.livy.rsc.RSCConf
-
-private case class RequestResponse(content: String, error: Boolean)
-
-// scalastyle:off println
-object SparkRInterpreter {
- private val LIVY_END_MARKER = "----LIVY_END_OF_COMMAND----"
- private val LIVY_ERROR_MARKER = "----LIVY_END_OF_ERROR----"
- private val PRINT_MARKER = f"""print("$LIVY_END_MARKER")"""
- private val EXPECTED_OUTPUT = f"""[1] "$LIVY_END_MARKER""""
-
- private val PLOT_REGEX = (
- "(" +
- "(?:bagplot)|" +
- "(?:barplot)|" +
- "(?:boxplot)|" +
- "(?:dotchart)|" +
- "(?:hist)|" +
- "(?:lines)|" +
- "(?:pie)|" +
- "(?:pie3D)|" +
- "(?:plot)|" +
- "(?:qqline)|" +
- "(?:qqnorm)|" +
- "(?:scatterplot)|" +
- "(?:scatterplot3d)|" +
- "(?:scatterplot\\.matrix)|" +
- "(?:splom)|" +
- "(?:stripchart)|" +
- "(?:vioplot)" +
- ")"
- ).r.unanchored
-
- def apply(conf: SparkConf): SparkRInterpreter = {
- val backendTimeout = sys.env.getOrElse("SPARKR_BACKEND_TIMEOUT", "120").toInt
- val mirror = universe.runtimeMirror(getClass.getClassLoader)
- val sparkRBackendClass = mirror.classLoader.loadClass("org.apache.spark.api.r.RBackend")
- val backendInstance = sparkRBackendClass.getDeclaredConstructor().newInstance()
-
- var sparkRBackendPort = 0
- val initialized = new Semaphore(0)
- // Launch a SparkR backend server for the R process to connect to
- val backendThread = new Thread("SparkR backend") {
- override def run(): Unit = {
- sparkRBackendPort = sparkRBackendClass.getMethod("init").invoke(backendInstance)
- .asInstanceOf[Int]
-
- initialized.release()
- sparkRBackendClass.getMethod("run").invoke(backendInstance)
- }
- }
-
- backendThread.setDaemon(true)
- backendThread.start()
- try {
- // Wait for RBackend initialization to finish
- initialized.tryAcquire(backendTimeout, TimeUnit.SECONDS)
- val rExec = conf.getOption("spark.r.shell.command")
- .orElse(sys.env.get("SPARKR_DRIVER_R"))
- .getOrElse("R")
-
- var packageDir = ""
- if (sys.env.getOrElse("SPARK_YARN_MODE", "") == "true") {
- packageDir = "./sparkr"
- } else {
- // local mode
- val rLibPath = new File(sys.env.getOrElse("SPARKR_PACKAGE_DIR",
- Seq(sys.env.getOrElse("SPARK_HOME", "."), "R", "lib").mkString(File.separator)))
- if (!ClientConf.TEST_MODE) {
- require(rLibPath.exists(), "Cannot find sparkr package directory.")
- packageDir = rLibPath.getAbsolutePath()
- }
- }
-
- val builder = new ProcessBuilder(Seq(rExec, "--slave @").asJava)
- val env = builder.environment()
- env.put("SPARK_HOME", sys.env.getOrElse("SPARK_HOME", "."))
- env.put("EXISTING_SPARKR_BACKEND_PORT", sparkRBackendPort.toString)
- env.put("SPARKR_PACKAGE_DIR", packageDir)
- env.put("R_PROFILE_USER",
- Seq(packageDir, "SparkR", "profile", "general.R").mkString(File.separator))
-
- builder.redirectErrorStream(true)
- val process = builder.start()
- new SparkRInterpreter(process, backendInstance, backendThread,
- conf.get("spark.livy.spark_major_version", "1"),
- conf.getBoolean("spark.repl.enableHiveContext", false))
- } catch {
- case e: Exception =>
- if (backendThread != null) {
- backendThread.interrupt()
- }
- throw e
- }
- }
-}
-
-class SparkRInterpreter(process: Process,
- backendInstance: Any,
- backendThread: Thread,
- val sparkMajorVersion: String,
- hiveEnabled: Boolean)
- extends ProcessInterpreter(process) {
- import SparkRInterpreter._
-
- implicit val formats = DefaultFormats
-
- private[this] var executionCount = 0
- override def kind: String = "sparkr"
- private[this] val isStarted = new CountDownLatch(1)
-
- final override protected def waitUntilReady(): Unit = {
- // Set the option to catch and ignore errors instead of halting.
- sendRequest("options(error = dump.frames)")
- if (!ClientConf.TEST_MODE) {
- sendRequest("library(SparkR)")
- if (sparkMajorVersion >= "2") {
- if (hiveEnabled) {
- sendRequest("spark <- SparkR::sparkR.session()")
- } else {
- sendRequest("spark <- SparkR::sparkR.session(enableHiveSupport=FALSE)")
- }
- sendRequest(
- """sc <- SparkR:::callJStatic("org.apache.spark.sql.api.r.SQLUtils",
- "getJavaSparkContext", spark)""")
- } else {
- sendRequest("sc <- sparkR.init()")
- if (hiveEnabled) {
- sendRequest("sqlContext <- sparkRHive.init(sc)")
- } else {
- sendRequest("sqlContext <- sparkRSQL.init(sc)")
- }
- }
- }
-
- isStarted.countDown()
- executionCount = 0
- }
-
- override protected def sendExecuteRequest(command: String): Interpreter.ExecuteResponse = {
- isStarted.await()
- var code = command
-
- // Create a image file if this command is trying to plot.
- val tempFile = PLOT_REGEX.findFirstIn(code).map { case _ =>
- val tempFile = Files.createTempFile("", ".png")
- val tempFileString = tempFile.toAbsolutePath
-
- code = f"""png("$tempFileString")\n$code\ndev.off()"""
-
- tempFile
- }
-
- try {
- val response = sendRequest(code)
-
- if (response.error) {
- Interpreter.ExecuteError("Error", response.content)
- } else {
- var content: JObject = TEXT_PLAIN -> response.content
-
- // If we rendered anything, pass along the last image.
- tempFile.foreach { case file =>
- val bytes = Files.readAllBytes(file)
- if (bytes.nonEmpty) {
- val image = Base64.encodeBase64String(bytes)
- content = content ~ (IMAGE_PNG -> image)
- }
- }
-
- Interpreter.ExecuteSuccess(content)
- }
-
- } catch {
- case e: Error =>
- Interpreter.ExecuteError("Error", e.output)
- case e: Exited =>
- Interpreter.ExecuteAborted(e.getMessage)
- } finally {
- tempFile.foreach(Files.delete)
- }
-
- }
-
- private def sendRequest(code: String): RequestResponse = {
- stdin.println(s"""tryCatch(eval(parse(text="${StringEscapeUtils.escapeJava(code)}"))
- |,error = function(e) sprintf("%s%s", e, "${LIVY_ERROR_MARKER}"))
- """.stripMargin)
- stdin.flush()
-
- stdin.println(PRINT_MARKER)
- stdin.flush()
-
- readTo(EXPECTED_OUTPUT, LIVY_ERROR_MARKER)
- }
-
- override protected def sendShutdownRequest() = {
- stdin.println("q()")
- stdin.flush()
-
- while (stdout.readLine() != null) {}
- }
-
- override def close(): Unit = {
- try {
- val closeMethod = backendInstance.getClass().getMethod("close")
- closeMethod.setAccessible(true)
- closeMethod.invoke(backendInstance)
-
- backendThread.interrupt()
- backendThread.join()
- } finally {
- super.close()
- }
- }
-
- @tailrec
- private def readTo(
- marker: String,
- errorMarker: String,
- output: StringBuilder = StringBuilder.newBuilder): RequestResponse = {
- var char = readChar(output)
-
- // Remove any ANSI color codes which match the pattern "\u001b\\[[0-9;]*[mG]".
- // It would be easier to do this with a regex, but unfortunately I don't see an easy way to do
- // without copying the StringBuilder into a string for each character.
- if (char == '\u001b') {
- if (readChar(output) == '[') {
- char = readDigits(output)
-
- if (char == 'm' || char == 'G') {
- output.delete(output.lastIndexOf('\u001b'), output.length)
- }
- }
- }
-
- if (output.endsWith(marker)) {
- var result = stripMarker(output.toString(), marker)
-
- if (result.endsWith(errorMarker + "\"")) {
- result = stripMarker(result, "\\n" + errorMarker)
- RequestResponse(result, error = true)
- } else {
- RequestResponse(result, error = false)
- }
- } else {
- readTo(marker, errorMarker, output)
- }
- }
-
- private def stripMarker(result: String, marker: String): String = {
- result.replace(marker, "")
- .stripPrefix("\n")
- .stripSuffix("\n")
- }
-
- private def readChar(output: StringBuilder): Char = {
- val byte = stdout.read()
- if (byte == -1) {
- throw new Exited(output.toString())
- } else {
- val char = byte.toChar
- output.append(char)
- char
- }
- }
-
- @tailrec
- private def readDigits(output: StringBuilder): Char = {
- val byte = stdout.read()
- if (byte == -1) {
- throw new Exited(output.toString())
- }
-
- val char = byte.toChar
-
- if (('0' to '9').contains(char)) {
- output.append(char)
- readDigits(output)
- } else {
- char
- }
- }
-
- private class Exited(val output: String) extends Exception {}
- private class Error(val output: String) extends Exception {}
-}
-// scalastyle:on println
[26/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/com/cloudera/livy/repl/package.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/package.scala b/repl/src/main/scala/com/cloudera/livy/repl/package.scala
deleted file mode 100644
index bf58ad4..0000000
--- a/repl/src/main/scala/com/cloudera/livy/repl/package.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy
-
-import org.json4s.JField
-
-package object repl {
- type MimeTypeMap = List[JField]
-
- val APPLICATION_JSON = "application/json"
- val APPLICATION_LIVY_TABLE_JSON = "application/vnd.livy.table.v1+json"
- val IMAGE_PNG = "image/png"
- val TEXT_PLAIN = "text/plain"
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/AbstractSparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/AbstractSparkInterpreter.scala b/repl/src/main/scala/org/apache/livy/repl/AbstractSparkInterpreter.scala
new file mode 100644
index 0000000..cf2cf35
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/AbstractSparkInterpreter.scala
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import java.io.ByteArrayOutputStream
+
+import scala.tools.nsc.interpreter.Results
+
+import org.apache.spark.rdd.RDD
+import org.json4s.DefaultFormats
+import org.json4s.Extraction
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+
+import org.apache.livy.Logging
+
+object AbstractSparkInterpreter {
+ private[repl] val KEEP_NEWLINE_REGEX = """(?<=\n)""".r
+ private val MAGIC_REGEX = "^%(\\w+)\\W*(.*)".r
+}
+
+abstract class AbstractSparkInterpreter extends Interpreter with Logging {
+ import AbstractSparkInterpreter._
+
+ private implicit def formats = DefaultFormats
+
+ protected val outputStream = new ByteArrayOutputStream()
+
+ final def kind: String = "spark"
+
+ protected def isStarted(): Boolean
+
+ protected def interpret(code: String): Results.Result
+
+ protected def valueOfTerm(name: String): Option[Any]
+
+ override protected[repl] def execute(code: String): Interpreter.ExecuteResponse =
+ restoreContextClassLoader {
+ require(isStarted())
+
+ executeLines(code.trim.split("\n").toList, Interpreter.ExecuteSuccess(JObject(
+ (TEXT_PLAIN, JString(""))
+ )))
+ }
+
+ private def executeMagic(magic: String, rest: String): Interpreter.ExecuteResponse = {
+ magic match {
+ case "json" => executeJsonMagic(rest)
+ case "table" => executeTableMagic(rest)
+ case _ =>
+ Interpreter.ExecuteError("UnknownMagic", f"Unknown magic command $magic")
+ }
+ }
+
+ private def executeJsonMagic(name: String): Interpreter.ExecuteResponse = {
+ try {
+ val value = valueOfTerm(name) match {
+ case Some(obj: RDD[_]) => obj.asInstanceOf[RDD[_]].take(10)
+ case Some(obj) => obj
+ case None => return Interpreter.ExecuteError("NameError", f"Value $name does not exist")
+ }
+
+ Interpreter.ExecuteSuccess(JObject(
+ (APPLICATION_JSON, Extraction.decompose(value))
+ ))
+ } catch {
+ case _: Throwable =>
+ Interpreter.ExecuteError("ValueError", "Failed to convert value into a JSON value")
+ }
+ }
+
+ private class TypesDoNotMatch extends Exception
+
+ private def convertTableType(value: JValue): String = {
+ value match {
+ case (JNothing | JNull) => "NULL_TYPE"
+ case JBool(_) => "BOOLEAN_TYPE"
+ case JString(_) => "STRING_TYPE"
+ case JInt(_) => "BIGINT_TYPE"
+ case JDouble(_) => "DOUBLE_TYPE"
+ case JDecimal(_) => "DECIMAL_TYPE"
+ case JArray(arr) =>
+ if (allSameType(arr.iterator)) {
+ "ARRAY_TYPE"
+ } else {
+ throw new TypesDoNotMatch
+ }
+ case JObject(obj) =>
+ if (allSameType(obj.iterator.map(_._2))) {
+ "MAP_TYPE"
+ } else {
+ throw new TypesDoNotMatch
+ }
+ }
+ }
+
+ private def allSameType(values: Iterator[JValue]): Boolean = {
+ if (values.hasNext) {
+ val type_name = convertTableType(values.next())
+ values.forall { case value => type_name.equals(convertTableType(value)) }
+ } else {
+ true
+ }
+ }
+
+ private def executeTableMagic(name: String): Interpreter.ExecuteResponse = {
+ val value = valueOfTerm(name) match {
+ case Some(obj: RDD[_]) => obj.asInstanceOf[RDD[_]].take(10)
+ case Some(obj) => obj
+ case None => return Interpreter.ExecuteError("NameError", f"Value $name does not exist")
+ }
+
+ extractTableFromJValue(Extraction.decompose(value))
+ }
+
+ private def extractTableFromJValue(value: JValue): Interpreter.ExecuteResponse = {
+ // Convert the value into JSON and map it to a table.
+ val rows: List[JValue] = value match {
+ case JArray(arr) => arr
+ case _ => List(value)
+ }
+
+ try {
+ val headers = scala.collection.mutable.Map[String, Map[String, String]]()
+
+ val data = rows.map { case row =>
+ val cols: List[JField] = row match {
+ case JArray(arr: List[JValue]) =>
+ arr.zipWithIndex.map { case (v, index) => JField(index.toString, v) }
+ case JObject(obj) => obj.sortBy(_._1)
+ case value: JValue => List(JField("0", value))
+ }
+
+ cols.map { case (k, v) =>
+ val typeName = convertTableType(v)
+
+ headers.get(k) match {
+ case Some(header) =>
+ if (header.get("type").get != typeName) {
+ throw new TypesDoNotMatch
+ }
+ case None =>
+ headers.put(k, Map(
+ "type" -> typeName,
+ "name" -> k
+ ))
+ }
+
+ v
+ }
+ }
+
+ Interpreter.ExecuteSuccess(
+ APPLICATION_LIVY_TABLE_JSON -> (
+ ("headers" -> headers.toSeq.sortBy(_._1).map(_._2)) ~ ("data" -> data)
+ ))
+ } catch {
+ case _: TypesDoNotMatch =>
+ Interpreter.ExecuteError("TypeError", "table rows have different types")
+ }
+ }
+
+ private def executeLines(
+ lines: List[String],
+ resultFromLastLine: Interpreter.ExecuteResponse): Interpreter.ExecuteResponse = {
+ lines match {
+ case Nil => resultFromLastLine
+ case head :: tail =>
+ val result = executeLine(head)
+
+ result match {
+ case Interpreter.ExecuteIncomplete() =>
+ tail match {
+ case Nil =>
+ // ExecuteIncomplete could be caused by an actual incomplete statements (e.g. "sc.")
+ // or statements with just comments.
+ // To distinguish them, reissue the same statement wrapped in { }.
+ // If it is an actual incomplete statement, the interpreter will return an error.
+ // If it is some comment, the interpreter will return success.
+ executeLine(s"{\n$head\n}") match {
+ case Interpreter.ExecuteIncomplete() | Interpreter.ExecuteError(_, _, _) =>
+ // Return the original error so users won't get confusing error message.
+ result
+ case _ => resultFromLastLine
+ }
+ case next :: nextTail =>
+ executeLines(head + "\n" + next :: nextTail, resultFromLastLine)
+ }
+ case Interpreter.ExecuteError(_, _, _) =>
+ result
+
+ case _ =>
+ executeLines(tail, result)
+ }
+ }
+ }
+
+ private def executeLine(code: String): Interpreter.ExecuteResponse = {
+ code match {
+ case MAGIC_REGEX(magic, rest) =>
+ executeMagic(magic, rest)
+ case _ =>
+ scala.Console.withOut(outputStream) {
+ interpret(code) match {
+ case Results.Success =>
+ Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> readStdout()
+ )
+ case Results.Incomplete => Interpreter.ExecuteIncomplete()
+ case Results.Error =>
+ val (ename, traceback) = parseError(readStdout())
+ Interpreter.ExecuteError("Error", ename, traceback)
+ }
+ }
+ }
+ }
+
+ protected[repl] def parseError(stdout: String): (String, Seq[String]) = {
+ // An example of Scala compile error message:
+ // <console>:27: error: type mismatch;
+ // found : Int
+ // required: Boolean
+
+ // An example of Scala runtime exception error message:
+ // java.lang.RuntimeException: message
+ // at .error(<console>:11)
+ // ... 32 elided
+
+ // Return the first line as ename. Lines following as traceback.
+
+ val lines = KEEP_NEWLINE_REGEX.split(stdout)
+ val ename = lines.headOption.map(_.trim).getOrElse("unknown error")
+ val traceback = lines.tail
+
+ (ename, traceback)
+ }
+
+ protected def restoreContextClassLoader[T](fn: => T): T = {
+ val currentClassLoader = Thread.currentThread().getContextClassLoader()
+ try {
+ fn
+ } finally {
+ Thread.currentThread().setContextClassLoader(currentClassLoader)
+ }
+ }
+
+ private def readStdout() = {
+ val output = outputStream.toString("UTF-8").trim
+ outputStream.reset()
+
+ output
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/BypassPySparkJob.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/BypassPySparkJob.scala b/repl/src/main/scala/org/apache/livy/repl/BypassPySparkJob.scala
new file mode 100644
index 0000000..6a9bdd1
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/BypassPySparkJob.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.repl
+
+import java.nio.charset.StandardCharsets
+
+import org.apache.livy.{Job, JobContext}
+
+class BypassPySparkJob(
+ serializedJob: Array[Byte],
+ replDriver: ReplDriver) extends Job[Array[Byte]] {
+
+ override def call(jc: JobContext): Array[Byte] = {
+ val interpreter = replDriver.interpreter
+ require(interpreter != null && interpreter.isInstanceOf[PythonInterpreter])
+ val pi = interpreter.asInstanceOf[PythonInterpreter]
+
+ val resultByteArray = pi.pysparkJobProcessor.processBypassJob(serializedJob)
+ val resultString = new String(resultByteArray, StandardCharsets.UTF_8)
+ if (resultString.startsWith("Client job error:")) {
+ throw new PythonJobException(resultString)
+ }
+ resultByteArray
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/Interpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/Interpreter.scala b/repl/src/main/scala/org/apache/livy/repl/Interpreter.scala
new file mode 100644
index 0000000..058b20b
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/Interpreter.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import org.apache.spark.SparkContext
+import org.json4s.JObject
+
+object Interpreter {
+ abstract class ExecuteResponse
+
+ case class ExecuteSuccess(content: JObject) extends ExecuteResponse
+ case class ExecuteError(ename: String,
+ evalue: String,
+ traceback: Seq[String] = Seq()) extends ExecuteResponse
+ case class ExecuteIncomplete() extends ExecuteResponse
+ case class ExecuteAborted(message: String) extends ExecuteResponse
+}
+
+trait Interpreter {
+ import Interpreter._
+
+ def kind: String
+
+ /**
+ * Start the Interpreter.
+ *
+ * @return A SparkContext
+ */
+ def start(): SparkContext
+
+ /**
+ * Execute the code and return the result, it may
+ * take some time to execute.
+ */
+ protected[repl] def execute(code: String): ExecuteResponse
+
+ /** Shut down the interpreter. */
+ def close(): Unit
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/ProcessInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/ProcessInterpreter.scala b/repl/src/main/scala/org/apache/livy/repl/ProcessInterpreter.scala
new file mode 100644
index 0000000..7995ba0
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/ProcessInterpreter.scala
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import java.io.{BufferedReader, InputStreamReader, IOException, PrintWriter}
+import java.util.concurrent.locks.ReentrantLock
+
+import scala.concurrent.Promise
+import scala.io.Source
+
+import org.apache.spark.SparkContext
+import org.json4s.JValue
+
+import org.apache.livy.{Logging, Utils}
+import org.apache.livy.client.common.ClientConf
+
+private sealed trait Request
+private case class ExecuteRequest(code: String, promise: Promise[JValue]) extends Request
+private case class ShutdownRequest(promise: Promise[Unit]) extends Request
+
+/**
+ * Abstract class that describes an interpreter that is running in a separate process.
+ *
+ * This type is not thread safe, so must be protected by a mutex.
+ *
+ * @param process
+ */
+abstract class ProcessInterpreter(process: Process)
+ extends Interpreter with Logging {
+ protected[this] val stdin = new PrintWriter(process.getOutputStream)
+ protected[this] val stdout = new BufferedReader(new InputStreamReader(process.getInputStream), 1)
+
+ override def start(): SparkContext = {
+ waitUntilReady()
+
+ if (ClientConf.TEST_MODE) {
+ null.asInstanceOf[SparkContext]
+ } else {
+ SparkContext.getOrCreate()
+ }
+ }
+
+ override protected[repl] def execute(code: String): Interpreter.ExecuteResponse = {
+ try {
+ sendExecuteRequest(code)
+ } catch {
+ case e: Throwable =>
+ Interpreter.ExecuteError(e.getClass.getName, e.getMessage)
+ }
+ }
+
+ override def close(): Unit = {
+ if (Utils.isProcessAlive(process)) {
+ logger.info("Shutting down process")
+ sendShutdownRequest()
+
+ try {
+ process.getInputStream.close()
+ process.getOutputStream.close()
+ } catch {
+ case _: IOException =>
+ }
+
+ try {
+ process.destroy()
+ } finally {
+ logger.info("process has been shut down")
+ }
+ }
+ }
+
+ protected def sendExecuteRequest(request: String): Interpreter.ExecuteResponse
+
+ protected def sendShutdownRequest(): Unit = {}
+
+ protected def waitUntilReady(): Unit
+
+ private[this] val stderrLock = new ReentrantLock()
+ private[this] var stderrLines = Seq[String]()
+
+ protected def takeErrorLines(): String = {
+ stderrLock.lock()
+ try {
+ val lines = stderrLines
+ stderrLines = Seq()
+ lines.mkString("\n")
+ } finally {
+ stderrLock.unlock()
+ }
+ }
+
+ private[this] val stderrThread = new Thread("process stderr thread") {
+ override def run() = {
+ val lines = Source.fromInputStream(process.getErrorStream).getLines()
+
+ for (line <- lines) {
+ stderrLock.lock()
+ try {
+ stderrLines :+= line
+ } finally {
+ stderrLock.unlock()
+ }
+ }
+ }
+ }
+
+ stderrThread.setDaemon(true)
+ stderrThread.start()
+
+ private[this] val processWatcherThread = new Thread("process watcher thread") {
+ override def run() = {
+ val exitCode = process.waitFor()
+ if (exitCode != 0) {
+ error(f"Process has died with $exitCode")
+ error(stderrLines.mkString("\n"))
+ }
+ }
+ }
+
+ processWatcherThread.setDaemon(true)
+ processWatcherThread.start()
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/PySparkJobProcessor.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/PySparkJobProcessor.scala b/repl/src/main/scala/org/apache/livy/repl/PySparkJobProcessor.scala
new file mode 100644
index 0000000..95b91cf
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/PySparkJobProcessor.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.repl
+
+trait PySparkJobProcessor {
+ def processBypassJob(job: Array[Byte]): Array[Byte]
+
+ def addFile(path: String)
+
+ def addPyFile(path: String)
+
+ def getLocalTmpDirPath: String
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala b/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
new file mode 100644
index 0000000..bfd5d76
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import java.io._
+import java.lang.ProcessBuilder.Redirect
+import java.lang.reflect.Proxy
+import java.nio.file.{Files, Paths}
+
+import scala.annotation.tailrec
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.JavaConverters._
+import scala.util.control.NonFatal
+
+import org.apache.spark.{SparkConf, SparkContext}
+import org.json4s.{DefaultFormats, JValue}
+import org.json4s.JsonAST.JObject
+import org.json4s.jackson.JsonMethods._
+import org.json4s.jackson.Serialization.write
+import py4j._
+import py4j.reflection.PythonProxyHandler
+
+import org.apache.livy.Logging
+import org.apache.livy.client.common.ClientConf
+import org.apache.livy.rsc.BaseProtocol
+import org.apache.livy.rsc.driver.BypassJobWrapper
+import org.apache.livy.sessions._
+
+// scalastyle:off println
+object PythonInterpreter extends Logging {
+
+ def apply(conf: SparkConf, kind: Kind): Interpreter = {
+ val pythonExec = kind match {
+ case PySpark() => sys.env.getOrElse("PYSPARK_PYTHON", "python")
+ case PySpark3() => sys.env.getOrElse("PYSPARK3_PYTHON", "python3")
+ case _ => throw new IllegalArgumentException(s"Unknown kind: $kind")
+ }
+
+ val gatewayServer = new GatewayServer(null, 0)
+ gatewayServer.start()
+
+ val builder = new ProcessBuilder(Seq(pythonExec, createFakeShell().toString).asJava)
+
+ val env = builder.environment()
+
+ val pythonPath = sys.env.getOrElse("PYTHONPATH", "")
+ .split(File.pathSeparator)
+ .++(if (!ClientConf.TEST_MODE) findPySparkArchives() else Nil)
+ .++(if (!ClientConf.TEST_MODE) findPyFiles() else Nil)
+
+ env.put("PYSPARK_PYTHON", pythonExec)
+ env.put("PYTHONPATH", pythonPath.mkString(File.pathSeparator))
+ env.put("PYTHONUNBUFFERED", "YES")
+ env.put("PYSPARK_GATEWAY_PORT", "" + gatewayServer.getListeningPort)
+ env.put("SPARK_HOME", sys.env.getOrElse("SPARK_HOME", "."))
+ env.put("LIVY_SPARK_MAJOR_VERSION", conf.get("spark.livy.spark_major_version", "1"))
+ builder.redirectError(Redirect.PIPE)
+ val process = builder.start()
+ new PythonInterpreter(process, gatewayServer, kind.toString)
+ }
+
+ private def findPySparkArchives(): Seq[String] = {
+ sys.env.get("PYSPARK_ARCHIVES_PATH")
+ .map(_.split(",").toSeq)
+ .getOrElse {
+ sys.env.get("SPARK_HOME").map { sparkHome =>
+ val pyLibPath = Seq(sparkHome, "python", "lib").mkString(File.separator)
+ val pyArchivesFile = new File(pyLibPath, "pyspark.zip")
+ require(pyArchivesFile.exists(),
+ "pyspark.zip not found; cannot run pyspark application in YARN mode.")
+
+ val py4jFile = Files.newDirectoryStream(Paths.get(pyLibPath), "py4j-*-src.zip")
+ .iterator()
+ .next()
+ .toFile
+
+ require(py4jFile.exists(),
+ "py4j-*-src.zip not found; cannot run pyspark application in YARN mode.")
+ Seq(pyArchivesFile.getAbsolutePath, py4jFile.getAbsolutePath)
+ }.getOrElse(Seq())
+ }
+ }
+
+ private def findPyFiles(): Seq[String] = {
+ val pyFiles = sys.props.getOrElse("spark.submit.pyFiles", "").split(",")
+
+ if (sys.env.getOrElse("SPARK_YARN_MODE", "") == "true") {
+ // In spark mode, these files have been localized into the current directory.
+ pyFiles.map { file =>
+ val name = new File(file).getName
+ new File(name).getAbsolutePath
+ }
+ } else {
+ pyFiles
+ }
+ }
+
+ private def createFakeShell(): File = {
+ val source: InputStream = getClass.getClassLoader.getResourceAsStream("fake_shell.py")
+
+ val file = Files.createTempFile("", "").toFile
+ file.deleteOnExit()
+
+ val sink = new FileOutputStream(file)
+ val buf = new Array[Byte](1024)
+ var n = source.read(buf)
+
+ while (n > 0) {
+ sink.write(buf, 0, n)
+ n = source.read(buf)
+ }
+
+ source.close()
+ sink.close()
+
+ file
+ }
+
+ private def initiatePy4jCallbackGateway(server: GatewayServer): PySparkJobProcessor = {
+ val f = server.getClass.getDeclaredField("gateway")
+ f.setAccessible(true)
+ val gateway = f.get(server).asInstanceOf[Gateway]
+ val command: String = "f" + Protocol.ENTRY_POINT_OBJECT_ID + ";" +
+ "org.apache.livy.repl.PySparkJobProcessor"
+ getPythonProxy(command, gateway).asInstanceOf[PySparkJobProcessor]
+ }
+
+ // This method is a hack to get around the classLoader issues faced in py4j 0.8.2.1 for
+ // dynamically adding jars to the driver. The change is to use the context classLoader instead
+ // of the system classLoader when initiating a new Proxy instance
+ // ISSUE - https://issues.apache.org/jira/browse/SPARK-6047
+ // FIX - https://github.com/bartdag/py4j/pull/196
+ private def getPythonProxy(commandPart: String, gateway: Gateway): Any = {
+ val proxyString = commandPart.substring(1, commandPart.length)
+ val parts = proxyString.split(";")
+ val length: Int = parts.length
+ val interfaces = ArrayBuffer.fill[Class[_]](length - 1){ null }
+ if (length < 2) {
+ throw new Py4JException("Invalid Python Proxy.")
+ }
+ else {
+ var proxy: Int = 1
+ while (proxy < length) {
+ try {
+ interfaces(proxy - 1) = Class.forName(parts(proxy))
+ if (!interfaces(proxy - 1).isInterface) {
+ throw new Py4JException("This class " + parts(proxy) +
+ " is not an interface and cannot be used as a Python Proxy.")
+ }
+ } catch {
+ case exception: ClassNotFoundException => {
+ throw new Py4JException("Invalid interface name: " + parts(proxy))
+ }
+ }
+ proxy += 1
+ }
+
+ val pythonProxyHandler = try {
+ classOf[PythonProxyHandler].getConstructor(classOf[String], classOf[Gateway])
+ .newInstance(parts(0), gateway)
+ } catch {
+ case NonFatal(e) =>
+ classOf[PythonProxyHandler].getConstructor(classOf[String],
+ Class.forName("py4j.CallbackClient"), classOf[Gateway])
+ .newInstance(parts(0), gateway.getCallbackClient, gateway)
+ }
+
+ Proxy.newProxyInstance(Thread.currentThread.getContextClassLoader,
+ interfaces.toArray, pythonProxyHandler.asInstanceOf[PythonProxyHandler])
+ }
+ }
+}
+
+private class PythonInterpreter(
+ process: Process,
+ gatewayServer: GatewayServer,
+ pyKind: String)
+ extends ProcessInterpreter(process)
+ with Logging
+{
+ implicit val formats = DefaultFormats
+
+ override def kind: String = pyKind
+
+ private[repl] val pysparkJobProcessor =
+ PythonInterpreter.initiatePy4jCallbackGateway(gatewayServer)
+
+ override def close(): Unit = {
+ try {
+ super.close()
+ } finally {
+ gatewayServer.shutdown()
+ }
+ }
+
+ @tailrec
+ final override protected def waitUntilReady(): Unit = {
+ val READY_REGEX = "READY\\(port=([0-9]+)\\)".r
+ stdout.readLine() match {
+ case null =>
+ case READY_REGEX(port) => updatePythonGatewayPort(port.toInt)
+ case _ => waitUntilReady()
+ }
+ }
+
+ override protected def sendExecuteRequest(code: String): Interpreter.ExecuteResponse = {
+ sendRequest(Map("msg_type" -> "execute_request", "content" -> Map("code" -> code))) match {
+ case Some(response) =>
+ assert((response \ "msg_type").extract[String] == "execute_reply")
+
+ val content = response \ "content"
+
+ (content \ "status").extract[String] match {
+ case "ok" =>
+ Interpreter.ExecuteSuccess((content \ "data").extract[JObject])
+ case "error" =>
+ val ename = (content \ "ename").extract[String]
+ val evalue = (content \ "evalue").extract[String]
+ val traceback = (content \ "traceback").extract[Seq[String]]
+
+ Interpreter.ExecuteError(ename, evalue, traceback)
+ case status =>
+ Interpreter.ExecuteError("Internal Error", f"Unknown status $status")
+ }
+ case None =>
+ Interpreter.ExecuteAborted(takeErrorLines())
+ }
+ }
+
+ override protected def sendShutdownRequest(): Unit = {
+ sendRequest(Map(
+ "msg_type" -> "shutdown_request",
+ "content" -> ()
+ )).foreach { case rep =>
+ warn(f"process failed to shut down while returning $rep")
+ }
+ }
+
+ private def sendRequest(request: Map[String, Any]): Option[JValue] = {
+ stdin.println(write(request))
+ stdin.flush()
+
+ Option(stdout.readLine()).map { case line =>
+ parse(line)
+ }
+ }
+
+ def addFile(path: String): Unit = {
+ pysparkJobProcessor.addFile(path)
+ }
+
+ def addPyFile(driver: ReplDriver, conf: SparkConf, path: String): Unit = {
+ val localCopyDir = new File(pysparkJobProcessor.getLocalTmpDirPath)
+ val localCopyFile = driver.copyFileToLocal(localCopyDir, path, SparkContext.getOrCreate(conf))
+ pysparkJobProcessor.addPyFile(localCopyFile.getPath)
+ if (path.endsWith(".jar")) {
+ driver.addLocalFileToClassLoader(localCopyFile)
+ }
+ }
+
+ private def updatePythonGatewayPort(port: Int): Unit = {
+ // The python gateway port can be 0 only when LivyConf.TEST_MODE is true
+ // Py4j 0.10 has different API signature for "getCallbackClient", use reflection to handle it.
+ if (port != 0) {
+ val callbackClient = gatewayServer.getClass
+ .getMethod("getCallbackClient")
+ .invoke(gatewayServer)
+
+ val field = Class.forName("py4j.CallbackClient").getDeclaredField("port")
+ field.setAccessible(true)
+ field.setInt(callbackClient, port.toInt)
+ }
+ }
+}
+
+case class PythonJobException(message: String) extends Exception(message) {}
+
+// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/ReplDriver.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/ReplDriver.scala b/repl/src/main/scala/org/apache/livy/repl/ReplDriver.scala
new file mode 100644
index 0000000..75966be
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/ReplDriver.scala
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
+import io.netty.channel.ChannelHandlerContext
+import org.apache.spark.SparkConf
+import org.apache.spark.api.java.JavaSparkContext
+
+import org.apache.livy.Logging
+import org.apache.livy.rsc.{BaseProtocol, ReplJobResults, RSCConf}
+import org.apache.livy.rsc.BaseProtocol.ReplState
+import org.apache.livy.rsc.driver._
+import org.apache.livy.rsc.rpc.Rpc
+import org.apache.livy.sessions._
+
+class ReplDriver(conf: SparkConf, livyConf: RSCConf)
+ extends RSCDriver(conf, livyConf)
+ with Logging {
+
+ private[repl] var session: Session = _
+
+ private val kind = Kind(livyConf.get(RSCConf.Entry.SESSION_KIND))
+
+ private[repl] var interpreter: Interpreter = _
+
+ override protected def initializeContext(): JavaSparkContext = {
+ interpreter = kind match {
+ case PySpark() => PythonInterpreter(conf, PySpark())
+ case PySpark3() =>
+ PythonInterpreter(conf, PySpark3())
+ case Spark() => new SparkInterpreter(conf)
+ case SparkR() => SparkRInterpreter(conf)
+ }
+ session = new Session(livyConf, interpreter, { s => broadcast(new ReplState(s.toString)) })
+
+ Option(Await.result(session.start(), Duration.Inf))
+ .map(new JavaSparkContext(_))
+ .orNull
+ }
+
+ override protected def shutdownContext(): Unit = {
+ if (session != null) {
+ try {
+ session.close()
+ } finally {
+ super.shutdownContext()
+ }
+ }
+ }
+
+ def handle(ctx: ChannelHandlerContext, msg: BaseProtocol.ReplJobRequest): Int = {
+ session.execute(msg.code)
+ }
+
+ def handle(ctx: ChannelHandlerContext, msg: BaseProtocol.CancelReplJobRequest): Unit = {
+ session.cancel(msg.id)
+ }
+
+ /**
+ * Return statement results. Results are sorted by statement id.
+ */
+ def handle(ctx: ChannelHandlerContext, msg: BaseProtocol.GetReplJobResults): ReplJobResults = {
+ val statements = if (msg.allResults) {
+ session.statements.values.toArray
+ } else {
+ assert(msg.from != null)
+ assert(msg.size != null)
+ if (msg.size == 1) {
+ session.statements.get(msg.from).toArray
+ } else {
+ val until = msg.from + msg.size
+ session.statements.filterKeys(id => id >= msg.from && id < until).values.toArray
+ }
+ }
+
+ // Update progress of statements when queried
+ statements.foreach { s =>
+ s.updateProgress(session.progressOfStatement(s.id))
+ }
+
+ new ReplJobResults(statements.sortBy(_.id))
+ }
+
+ override protected def createWrapper(msg: BaseProtocol.BypassJobRequest): BypassJobWrapper = {
+ kind match {
+ case PySpark() | PySpark3() => new BypassJobWrapper(this, msg.id,
+ new BypassPySparkJob(msg.serializedJob, this))
+ case _ => super.createWrapper(msg)
+ }
+ }
+
+ override protected def addFile(path: String): Unit = {
+ require(interpreter != null)
+ interpreter match {
+ case pi: PythonInterpreter => pi.addFile(path)
+ case _ => super.addFile(path)
+ }
+ }
+
+ override protected def addJarOrPyFile(path: String): Unit = {
+ require(interpreter != null)
+ interpreter match {
+ case pi: PythonInterpreter => pi.addPyFile(this, conf, path)
+ case _ => super.addJarOrPyFile(path)
+ }
+ }
+
+ override protected def onClientAuthenticated(client: Rpc): Unit = {
+ if (session != null) {
+ client.call(new ReplState(session.state.toString))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/Session.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/Session.scala b/repl/src/main/scala/org/apache/livy/repl/Session.scala
new file mode 100644
index 0000000..40176ea
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/Session.scala
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import java.util.{LinkedHashMap => JLinkedHashMap}
+import java.util.Map.Entry
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicInteger
+
+import scala.collection.JavaConverters._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
+
+import org.apache.spark.SparkContext
+import org.json4s.jackson.JsonMethods.{compact, render}
+import org.json4s.DefaultFormats
+import org.json4s.JsonDSL._
+
+import org.apache.livy.Logging
+import org.apache.livy.rsc.RSCConf
+import org.apache.livy.rsc.driver.{Statement, StatementState}
+import org.apache.livy.sessions._
+
+object Session {
+ val STATUS = "status"
+ val OK = "ok"
+ val ERROR = "error"
+ val EXECUTION_COUNT = "execution_count"
+ val DATA = "data"
+ val ENAME = "ename"
+ val EVALUE = "evalue"
+ val TRACEBACK = "traceback"
+}
+
+class Session(
+ livyConf: RSCConf,
+ interpreter: Interpreter,
+ stateChangedCallback: SessionState => Unit = { _ => })
+ extends Logging {
+ import Session._
+
+ private val interpreterExecutor = ExecutionContext.fromExecutorService(
+ Executors.newSingleThreadExecutor())
+
+ private val cancelExecutor = ExecutionContext.fromExecutorService(
+ Executors.newSingleThreadExecutor())
+
+ private implicit val formats = DefaultFormats
+
+ @volatile private[repl] var _sc: Option[SparkContext] = None
+
+ private var _state: SessionState = SessionState.NotStarted()
+
+ // Number of statements kept in driver's memory
+ private val numRetainedStatements = livyConf.getInt(RSCConf.Entry.RETAINED_STATEMENT_NUMBER)
+
+ private val _statements = new JLinkedHashMap[Int, Statement] {
+ protected override def removeEldestEntry(eldest: Entry[Int, Statement]): Boolean = {
+ size() > numRetainedStatements
+ }
+ }.asScala
+
+ private val newStatementId = new AtomicInteger(0)
+
+ stateChangedCallback(_state)
+
+ def start(): Future[SparkContext] = {
+ val future = Future {
+ changeState(SessionState.Starting())
+ val sc = interpreter.start()
+ _sc = Option(sc)
+ changeState(SessionState.Idle())
+ sc
+ }(interpreterExecutor)
+
+ future.onFailure { case _ => changeState(SessionState.Error()) }(interpreterExecutor)
+ future
+ }
+
+ def kind: String = interpreter.kind
+
+ def state: SessionState = _state
+
+ def statements: collection.Map[Int, Statement] = _statements.synchronized {
+ _statements.toMap
+ }
+
+ def execute(code: String): Int = {
+ val statementId = newStatementId.getAndIncrement()
+ val statement = new Statement(statementId, code, StatementState.Waiting, null)
+ _statements.synchronized { _statements(statementId) = statement }
+
+ Future {
+ setJobGroup(statementId)
+ statement.compareAndTransit(StatementState.Waiting, StatementState.Running)
+
+ if (statement.state.get() == StatementState.Running) {
+ statement.output = executeCode(statementId, code)
+ }
+
+ statement.compareAndTransit(StatementState.Running, StatementState.Available)
+ statement.compareAndTransit(StatementState.Cancelling, StatementState.Cancelled)
+ statement.updateProgress(1.0)
+ }(interpreterExecutor)
+
+ statementId
+ }
+
+ def cancel(statementId: Int): Unit = {
+ val statementOpt = _statements.synchronized { _statements.get(statementId) }
+ if (statementOpt.isEmpty) {
+ return
+ }
+
+ val statement = statementOpt.get
+ if (statement.state.get().isOneOf(
+ StatementState.Available, StatementState.Cancelled, StatementState.Cancelling)) {
+ return
+ } else {
+ // statement 1 is running and statement 2 is waiting. User cancels
+ // statement 2 then cancels statement 1. The 2nd cancel call will loop and block the 1st
+ // cancel call since cancelExecutor is single threaded. To avoid this, set the statement
+ // state to cancelled when cancelling a waiting statement.
+ statement.compareAndTransit(StatementState.Waiting, StatementState.Cancelled)
+ statement.compareAndTransit(StatementState.Running, StatementState.Cancelling)
+ }
+
+ info(s"Cancelling statement $statementId...")
+
+ Future {
+ val deadline = livyConf.getTimeAsMs(RSCConf.Entry.JOB_CANCEL_TIMEOUT).millis.fromNow
+
+ while (statement.state.get() == StatementState.Cancelling) {
+ if (deadline.isOverdue()) {
+ info(s"Failed to cancel statement $statementId.")
+ statement.compareAndTransit(StatementState.Cancelling, StatementState.Cancelled)
+ } else {
+ _sc.foreach(_.cancelJobGroup(statementId.toString))
+ if (statement.state.get() == StatementState.Cancelling) {
+ Thread.sleep(livyConf.getTimeAsMs(RSCConf.Entry.JOB_CANCEL_TRIGGER_INTERVAL))
+ }
+ }
+ }
+
+ if (statement.state.get() == StatementState.Cancelled) {
+ info(s"Statement $statementId cancelled.")
+ }
+ }(cancelExecutor)
+ }
+
+ def close(): Unit = {
+ interpreterExecutor.shutdown()
+ cancelExecutor.shutdown()
+ interpreter.close()
+ }
+
+ /**
+ * Get the current progress of given statement id.
+ */
+ def progressOfStatement(stmtId: Int): Double = {
+ val jobGroup = statementIdToJobGroup(stmtId)
+
+ _sc.map { sc =>
+ val jobIds = sc.statusTracker.getJobIdsForGroup(jobGroup)
+ val jobs = jobIds.flatMap { id => sc.statusTracker.getJobInfo(id) }
+ val stages = jobs.flatMap { job =>
+ job.stageIds().flatMap(sc.statusTracker.getStageInfo)
+ }
+
+ val taskCount = stages.map(_.numTasks).sum
+ val completedTaskCount = stages.map(_.numCompletedTasks).sum
+ if (taskCount == 0) {
+ 0.0
+ } else {
+ completedTaskCount.toDouble / taskCount
+ }
+ }.getOrElse(0.0)
+ }
+
+ private def changeState(newState: SessionState): Unit = {
+ synchronized {
+ _state = newState
+ }
+ stateChangedCallback(newState)
+ }
+
+ private def executeCode(executionCount: Int, code: String): String = {
+ changeState(SessionState.Busy())
+
+ def transitToIdle() = {
+ val executingLastStatement = executionCount == newStatementId.intValue() - 1
+ if (_statements.isEmpty || executingLastStatement) {
+ changeState(SessionState.Idle())
+ }
+ }
+
+ val resultInJson = try {
+ interpreter.execute(code) match {
+ case Interpreter.ExecuteSuccess(data) =>
+ transitToIdle()
+
+ (STATUS -> OK) ~
+ (EXECUTION_COUNT -> executionCount) ~
+ (DATA -> data)
+
+ case Interpreter.ExecuteIncomplete() =>
+ transitToIdle()
+
+ (STATUS -> ERROR) ~
+ (EXECUTION_COUNT -> executionCount) ~
+ (ENAME -> "Error") ~
+ (EVALUE -> "incomplete statement") ~
+ (TRACEBACK -> Seq.empty[String])
+
+ case Interpreter.ExecuteError(ename, evalue, traceback) =>
+ transitToIdle()
+
+ (STATUS -> ERROR) ~
+ (EXECUTION_COUNT -> executionCount) ~
+ (ENAME -> ename) ~
+ (EVALUE -> evalue) ~
+ (TRACEBACK -> traceback)
+
+ case Interpreter.ExecuteAborted(message) =>
+ changeState(SessionState.Error())
+
+ (STATUS -> ERROR) ~
+ (EXECUTION_COUNT -> executionCount) ~
+ (ENAME -> "Error") ~
+ (EVALUE -> f"Interpreter died:\n$message") ~
+ (TRACEBACK -> Seq.empty[String])
+ }
+ } catch {
+ case e: Throwable =>
+ error("Exception when executing code", e)
+
+ transitToIdle()
+
+ (STATUS -> ERROR) ~
+ (EXECUTION_COUNT -> executionCount) ~
+ (ENAME -> f"Internal Error: ${e.getClass.getName}") ~
+ (EVALUE -> e.getMessage) ~
+ (TRACEBACK -> Seq.empty[String])
+ }
+
+ compact(render(resultInJson))
+ }
+
+ private def setJobGroup(statementId: Int): String = {
+ val jobGroup = statementIdToJobGroup(statementId)
+ val cmd = Kind(interpreter.kind) match {
+ case Spark() =>
+ // A dummy value to avoid automatic value binding in scala REPL.
+ s"""val _livyJobGroup$jobGroup = sc.setJobGroup("$jobGroup",""" +
+ s""""Job group for statement $jobGroup")"""
+ case PySpark() | PySpark3() =>
+ s"""sc.setJobGroup("$jobGroup", "Job group for statement $jobGroup")"""
+ case SparkR() =>
+ interpreter.asInstanceOf[SparkRInterpreter].sparkMajorVersion match {
+ case "1" =>
+ s"""setJobGroup(sc, "$jobGroup", "Job group for statement $jobGroup", """ +
+ "FALSE)"
+ case "2" =>
+ s"""setJobGroup("$jobGroup", "Job group for statement $jobGroup", FALSE)"""
+ }
+ }
+ // Set the job group
+ executeCode(statementId, cmd)
+ }
+
+ private def statementIdToJobGroup(statementId: Int): String = {
+ statementId.toString
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/SparkContextInitializer.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/SparkContextInitializer.scala b/repl/src/main/scala/org/apache/livy/repl/SparkContextInitializer.scala
new file mode 100644
index 0000000..a4a57f9
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/SparkContextInitializer.scala
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+import org.apache.livy.Logging
+
+/**
+ * A mixin trait for Spark entry point creation. This trait exists two different code path
+ * separately for Spark1 and Spark2, depends on whether SparkSession exists or not.
+ */
+trait SparkContextInitializer extends Logging {
+ self: SparkInterpreter =>
+
+ def createSparkContext(conf: SparkConf): Unit = {
+ if (isSparkSessionPresent()) {
+ spark2CreateContext(conf)
+ } else {
+ spark1CreateContext(conf)
+ }
+ }
+
+ private def spark1CreateContext(conf: SparkConf): Unit = {
+ sparkContext = SparkContext.getOrCreate(conf)
+ var sqlContext: Object = null
+
+ if (conf.getBoolean("spark.repl.enableHiveContext", false)) {
+ try {
+ val loader = Option(Thread.currentThread().getContextClassLoader)
+ .getOrElse(getClass.getClassLoader)
+ if (loader.getResource("hive-site.xml") == null) {
+ warn("livy.repl.enable-hive-context is true but no hive-site.xml found on classpath.")
+ }
+
+ sqlContext = Class.forName("org.apache.spark.sql.hive.HiveContext")
+ .getConstructor(classOf[SparkContext]).newInstance(sparkContext).asInstanceOf[Object]
+ info("Created sql context (with Hive support).")
+ } catch {
+ case _: NoClassDefFoundError =>
+ sqlContext = Class.forName("org.apache.spark.sql.SQLContext")
+ .getConstructor(classOf[SparkContext]).newInstance(sparkContext).asInstanceOf[Object]
+ info("Created sql context.")
+ }
+ } else {
+ sqlContext = Class.forName("org.apache.spark.sql.SQLContext")
+ .getConstructor(classOf[SparkContext]).newInstance(sparkContext).asInstanceOf[Object]
+ info("Created sql context.")
+ }
+
+ bind("sc", "org.apache.spark.SparkContext", sparkContext, List("""@transient"""))
+ bind("sqlContext", sqlContext.getClass.getCanonicalName, sqlContext, List("""@transient"""))
+
+ execute("import org.apache.spark.SparkContext._")
+ execute("import sqlContext.implicits._")
+ execute("import sqlContext.sql")
+ execute("import org.apache.spark.sql.functions._")
+ }
+
+ private def spark2CreateContext(conf: SparkConf): Unit = {
+ val sparkClz = Class.forName("org.apache.spark.sql.SparkSession$")
+ val sparkObj = sparkClz.getField("MODULE$").get(null)
+
+ val builderMethod = sparkClz.getMethod("builder")
+ val builder = builderMethod.invoke(sparkObj)
+ builder.getClass.getMethod("config", classOf[SparkConf]).invoke(builder, conf)
+
+ var spark: Object = null
+ if (conf.get("spark.sql.catalogImplementation", "in-memory").toLowerCase == "hive") {
+ if (sparkClz.getMethod("hiveClassesArePresent").invoke(sparkObj).asInstanceOf[Boolean]) {
+ val loader = Option(Thread.currentThread().getContextClassLoader)
+ .getOrElse(getClass.getClassLoader)
+ if (loader.getResource("hive-site.xml") == null) {
+ warn("livy.repl.enable-hive-context is true but no hive-site.xml found on classpath.")
+ }
+
+ builder.getClass.getMethod("enableHiveSupport").invoke(builder)
+ spark = builder.getClass.getMethod("getOrCreate").invoke(builder)
+ info("Created Spark session (with Hive support).")
+ } else {
+ spark = builder.getClass.getMethod("getOrCreate").invoke(builder)
+ info("Created Spark session.")
+ }
+ } else {
+ spark = builder.getClass.getMethod("getOrCreate").invoke(builder)
+ info("Created Spark session.")
+ }
+
+ sparkContext = spark.getClass.getMethod("sparkContext").invoke(spark)
+ .asInstanceOf[SparkContext]
+
+ bind("spark", spark.getClass.getCanonicalName, spark, List("""@transient"""))
+ bind("sc", "org.apache.spark.SparkContext", sparkContext, List("""@transient"""))
+
+ execute("import org.apache.spark.SparkContext._")
+ execute("import spark.implicits._")
+ execute("import spark.sql")
+ execute("import org.apache.spark.sql.functions._")
+ }
+
+ private def isSparkSessionPresent(): Boolean = {
+ try {
+ Class.forName("org.apache.spark.sql.SparkSession")
+ true
+ } catch {
+ case _: ClassNotFoundException | _: NoClassDefFoundError => false
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/SparkRInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/SparkRInterpreter.scala b/repl/src/main/scala/org/apache/livy/repl/SparkRInterpreter.scala
new file mode 100644
index 0000000..b745861
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/SparkRInterpreter.scala
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import java.io.{File, FileOutputStream}
+import java.lang.ProcessBuilder.Redirect
+import java.nio.file.Files
+import java.util.concurrent.{CountDownLatch, Semaphore, TimeUnit}
+
+import scala.annotation.tailrec
+import scala.collection.JavaConverters._
+import scala.reflect.runtime.universe
+
+import org.apache.commons.codec.binary.Base64
+import org.apache.commons.lang.StringEscapeUtils
+import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.util.{ChildFirstURLClassLoader, MutableURLClassLoader, Utils}
+import org.json4s._
+import org.json4s.JsonDSL._
+
+import org.apache.livy.client.common.ClientConf
+import org.apache.livy.rsc.RSCConf
+
+private case class RequestResponse(content: String, error: Boolean)
+
+// scalastyle:off println
+object SparkRInterpreter {
+ private val LIVY_END_MARKER = "----LIVY_END_OF_COMMAND----"
+ private val LIVY_ERROR_MARKER = "----LIVY_END_OF_ERROR----"
+ private val PRINT_MARKER = f"""print("$LIVY_END_MARKER")"""
+ private val EXPECTED_OUTPUT = f"""[1] "$LIVY_END_MARKER""""
+
+ private val PLOT_REGEX = (
+ "(" +
+ "(?:bagplot)|" +
+ "(?:barplot)|" +
+ "(?:boxplot)|" +
+ "(?:dotchart)|" +
+ "(?:hist)|" +
+ "(?:lines)|" +
+ "(?:pie)|" +
+ "(?:pie3D)|" +
+ "(?:plot)|" +
+ "(?:qqline)|" +
+ "(?:qqnorm)|" +
+ "(?:scatterplot)|" +
+ "(?:scatterplot3d)|" +
+ "(?:scatterplot\\.matrix)|" +
+ "(?:splom)|" +
+ "(?:stripchart)|" +
+ "(?:vioplot)" +
+ ")"
+ ).r.unanchored
+
+ def apply(conf: SparkConf): SparkRInterpreter = {
+ val backendTimeout = sys.env.getOrElse("SPARKR_BACKEND_TIMEOUT", "120").toInt
+ val mirror = universe.runtimeMirror(getClass.getClassLoader)
+ val sparkRBackendClass = mirror.classLoader.loadClass("org.apache.spark.api.r.RBackend")
+ val backendInstance = sparkRBackendClass.getDeclaredConstructor().newInstance()
+
+ var sparkRBackendPort = 0
+ val initialized = new Semaphore(0)
+ // Launch a SparkR backend server for the R process to connect to
+ val backendThread = new Thread("SparkR backend") {
+ override def run(): Unit = {
+ sparkRBackendPort = sparkRBackendClass.getMethod("init").invoke(backendInstance)
+ .asInstanceOf[Int]
+
+ initialized.release()
+ sparkRBackendClass.getMethod("run").invoke(backendInstance)
+ }
+ }
+
+ backendThread.setDaemon(true)
+ backendThread.start()
+ try {
+ // Wait for RBackend initialization to finish
+ initialized.tryAcquire(backendTimeout, TimeUnit.SECONDS)
+ val rExec = conf.getOption("spark.r.shell.command")
+ .orElse(sys.env.get("SPARKR_DRIVER_R"))
+ .getOrElse("R")
+
+ var packageDir = ""
+ if (sys.env.getOrElse("SPARK_YARN_MODE", "") == "true") {
+ packageDir = "./sparkr"
+ } else {
+ // local mode
+ val rLibPath = new File(sys.env.getOrElse("SPARKR_PACKAGE_DIR",
+ Seq(sys.env.getOrElse("SPARK_HOME", "."), "R", "lib").mkString(File.separator)))
+ if (!ClientConf.TEST_MODE) {
+ require(rLibPath.exists(), "Cannot find sparkr package directory.")
+ packageDir = rLibPath.getAbsolutePath()
+ }
+ }
+
+ val builder = new ProcessBuilder(Seq(rExec, "--slave @").asJava)
+ val env = builder.environment()
+ env.put("SPARK_HOME", sys.env.getOrElse("SPARK_HOME", "."))
+ env.put("EXISTING_SPARKR_BACKEND_PORT", sparkRBackendPort.toString)
+ env.put("SPARKR_PACKAGE_DIR", packageDir)
+ env.put("R_PROFILE_USER",
+ Seq(packageDir, "SparkR", "profile", "general.R").mkString(File.separator))
+
+ builder.redirectErrorStream(true)
+ val process = builder.start()
+ new SparkRInterpreter(process, backendInstance, backendThread,
+ conf.get("spark.livy.spark_major_version", "1"),
+ conf.getBoolean("spark.repl.enableHiveContext", false))
+ } catch {
+ case e: Exception =>
+ if (backendThread != null) {
+ backendThread.interrupt()
+ }
+ throw e
+ }
+ }
+}
+
+class SparkRInterpreter(process: Process,
+ backendInstance: Any,
+ backendThread: Thread,
+ val sparkMajorVersion: String,
+ hiveEnabled: Boolean)
+ extends ProcessInterpreter(process) {
+ import SparkRInterpreter._
+
+ implicit val formats = DefaultFormats
+
+ private[this] var executionCount = 0
+ override def kind: String = "sparkr"
+ private[this] val isStarted = new CountDownLatch(1)
+
+ final override protected def waitUntilReady(): Unit = {
+ // Set the option to catch and ignore errors instead of halting.
+ sendRequest("options(error = dump.frames)")
+ if (!ClientConf.TEST_MODE) {
+ sendRequest("library(SparkR)")
+ if (sparkMajorVersion >= "2") {
+ if (hiveEnabled) {
+ sendRequest("spark <- SparkR::sparkR.session()")
+ } else {
+ sendRequest("spark <- SparkR::sparkR.session(enableHiveSupport=FALSE)")
+ }
+ sendRequest(
+ """sc <- SparkR:::callJStatic("org.apache.spark.sql.api.r.SQLUtils",
+ "getJavaSparkContext", spark)""")
+ } else {
+ sendRequest("sc <- sparkR.init()")
+ if (hiveEnabled) {
+ sendRequest("sqlContext <- sparkRHive.init(sc)")
+ } else {
+ sendRequest("sqlContext <- sparkRSQL.init(sc)")
+ }
+ }
+ }
+
+ isStarted.countDown()
+ executionCount = 0
+ }
+
+ override protected def sendExecuteRequest(command: String): Interpreter.ExecuteResponse = {
+ isStarted.await()
+ var code = command
+
+ // Create a image file if this command is trying to plot.
+ val tempFile = PLOT_REGEX.findFirstIn(code).map { case _ =>
+ val tempFile = Files.createTempFile("", ".png")
+ val tempFileString = tempFile.toAbsolutePath
+
+ code = f"""png("$tempFileString")\n$code\ndev.off()"""
+
+ tempFile
+ }
+
+ try {
+ val response = sendRequest(code)
+
+ if (response.error) {
+ Interpreter.ExecuteError("Error", response.content)
+ } else {
+ var content: JObject = TEXT_PLAIN -> response.content
+
+ // If we rendered anything, pass along the last image.
+ tempFile.foreach { case file =>
+ val bytes = Files.readAllBytes(file)
+ if (bytes.nonEmpty) {
+ val image = Base64.encodeBase64String(bytes)
+ content = content ~ (IMAGE_PNG -> image)
+ }
+ }
+
+ Interpreter.ExecuteSuccess(content)
+ }
+
+ } catch {
+ case e: Error =>
+ Interpreter.ExecuteError("Error", e.output)
+ case e: Exited =>
+ Interpreter.ExecuteAborted(e.getMessage)
+ } finally {
+ tempFile.foreach(Files.delete)
+ }
+
+ }
+
+ private def sendRequest(code: String): RequestResponse = {
+ stdin.println(s"""tryCatch(eval(parse(text="${StringEscapeUtils.escapeJava(code)}"))
+ |,error = function(e) sprintf("%s%s", e, "${LIVY_ERROR_MARKER}"))
+ """.stripMargin)
+ stdin.flush()
+
+ stdin.println(PRINT_MARKER)
+ stdin.flush()
+
+ readTo(EXPECTED_OUTPUT, LIVY_ERROR_MARKER)
+ }
+
+ override protected def sendShutdownRequest() = {
+ stdin.println("q()")
+ stdin.flush()
+
+ while (stdout.readLine() != null) {}
+ }
+
+ override def close(): Unit = {
+ try {
+ val closeMethod = backendInstance.getClass().getMethod("close")
+ closeMethod.setAccessible(true)
+ closeMethod.invoke(backendInstance)
+
+ backendThread.interrupt()
+ backendThread.join()
+ } finally {
+ super.close()
+ }
+ }
+
+ @tailrec
+ private def readTo(
+ marker: String,
+ errorMarker: String,
+ output: StringBuilder = StringBuilder.newBuilder): RequestResponse = {
+ var char = readChar(output)
+
+ // Remove any ANSI color codes which match the pattern "\u001b\\[[0-9;]*[mG]".
+ // It would be easier to do this with a regex, but unfortunately I don't see an easy way to do
+ // without copying the StringBuilder into a string for each character.
+ if (char == '\u001b') {
+ if (readChar(output) == '[') {
+ char = readDigits(output)
+
+ if (char == 'm' || char == 'G') {
+ output.delete(output.lastIndexOf('\u001b'), output.length)
+ }
+ }
+ }
+
+ if (output.endsWith(marker)) {
+ var result = stripMarker(output.toString(), marker)
+
+ if (result.endsWith(errorMarker + "\"")) {
+ result = stripMarker(result, "\\n" + errorMarker)
+ RequestResponse(result, error = true)
+ } else {
+ RequestResponse(result, error = false)
+ }
+ } else {
+ readTo(marker, errorMarker, output)
+ }
+ }
+
+ private def stripMarker(result: String, marker: String): String = {
+ result.replace(marker, "")
+ .stripPrefix("\n")
+ .stripSuffix("\n")
+ }
+
+ private def readChar(output: StringBuilder): Char = {
+ val byte = stdout.read()
+ if (byte == -1) {
+ throw new Exited(output.toString())
+ } else {
+ val char = byte.toChar
+ output.append(char)
+ char
+ }
+ }
+
+ @tailrec
+ private def readDigits(output: StringBuilder): Char = {
+ val byte = stdout.read()
+ if (byte == -1) {
+ throw new Exited(output.toString())
+ }
+
+ val char = byte.toChar
+
+ if (('0' to '9').contains(char)) {
+ output.append(char)
+ readDigits(output)
+ } else {
+ char
+ }
+ }
+
+ private class Exited(val output: String) extends Exception {}
+ private class Error(val output: String) extends Exception {}
+}
+// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/main/scala/org/apache/livy/repl/package.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/package.scala b/repl/src/main/scala/org/apache/livy/repl/package.scala
new file mode 100644
index 0000000..7bc9fe9
--- /dev/null
+++ b/repl/src/main/scala/org/apache/livy/repl/package.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy
+
+import org.json4s.JField
+
+package object repl {
+ type MimeTypeMap = List[JField]
+
+ val APPLICATION_JSON = "application/json"
+ val APPLICATION_LIVY_TABLE_JSON = "application/vnd.livy.table.v1+json"
+ val IMAGE_PNG = "image/png"
+ val TEXT_PLAIN = "text/plain"
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/com/cloudera/livy/repl/BaseInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/com/cloudera/livy/repl/BaseInterpreterSpec.scala b/repl/src/test/scala/com/cloudera/livy/repl/BaseInterpreterSpec.scala
deleted file mode 100644
index 0c410d2..0000000
--- a/repl/src/test/scala/com/cloudera/livy/repl/BaseInterpreterSpec.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import org.scalatest.{FlatSpec, Matchers}
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-
-abstract class BaseInterpreterSpec extends FlatSpec with Matchers with LivyBaseUnitTestSuite {
-
- def createInterpreter(): Interpreter
-
- def withInterpreter(testCode: Interpreter => Any): Unit = {
- val interpreter = createInterpreter()
- try {
- interpreter.start()
- testCode(interpreter)
- } finally {
- interpreter.close()
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/com/cloudera/livy/repl/BaseSessionSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/com/cloudera/livy/repl/BaseSessionSpec.scala b/repl/src/test/scala/com/cloudera/livy/repl/BaseSessionSpec.scala
deleted file mode 100644
index 59db4b3..0000000
--- a/repl/src/test/scala/com/cloudera/livy/repl/BaseSessionSpec.scala
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import java.util.Properties
-import java.util.concurrent.atomic.AtomicInteger
-
-import scala.concurrent.Await
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import org.json4s._
-import org.scalatest.{FlatSpec, Matchers}
-import org.scalatest.concurrent.Eventually._
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-import com.cloudera.livy.rsc.RSCConf
-import com.cloudera.livy.rsc.driver.{Statement, StatementState}
-import com.cloudera.livy.sessions.SessionState
-
-abstract class BaseSessionSpec extends FlatSpec with Matchers with LivyBaseUnitTestSuite {
-
- implicit val formats = DefaultFormats
-
- private val rscConf = new RSCConf(new Properties())
-
- protected def execute(session: Session)(code: String): Statement = {
- val id = session.execute(code)
- eventually(timeout(30 seconds), interval(100 millis)) {
- val s = session.statements(id)
- s.state.get() shouldBe StatementState.Available
- s
- }
- }
-
- protected def withSession(testCode: Session => Any): Unit = {
- val stateChangedCalled = new AtomicInteger()
- val session =
- new Session(rscConf, createInterpreter(), { _ => stateChangedCalled.incrementAndGet() })
- try {
- // Session's constructor should fire an initial state change event.
- stateChangedCalled.intValue() shouldBe 1
- Await.ready(session.start(), 30 seconds)
- assert(session.state === SessionState.Idle())
- // There should be at least 1 state change event fired when session transits to idle.
- stateChangedCalled.intValue() should (be > 1)
- testCode(session)
- } finally {
- session.close()
- }
- }
-
- protected def createInterpreter(): Interpreter
-
- it should "start in the starting or idle state" in {
- val session = new Session(rscConf, createInterpreter())
- val future = session.start()
- try {
- eventually(timeout(30 seconds), interval(100 millis)) {
- session.state should (equal (SessionState.Starting()) or equal (SessionState.Idle()))
- }
- Await.ready(future, 60 seconds)
- } finally {
- session.close()
- }
- }
-
- it should "eventually become the idle state" in withSession { session =>
- session.state should equal (SessionState.Idle())
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/com/cloudera/livy/repl/PythonInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/com/cloudera/livy/repl/PythonInterpreterSpec.scala b/repl/src/test/scala/com/cloudera/livy/repl/PythonInterpreterSpec.scala
deleted file mode 100644
index 7917b90..0000000
--- a/repl/src/test/scala/com/cloudera/livy/repl/PythonInterpreterSpec.scala
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import org.apache.spark.SparkConf
-import org.json4s.{DefaultFormats, JNull, JValue}
-import org.json4s.JsonDSL._
-import org.scalatest._
-
-import com.cloudera.livy.rsc.RSCConf
-import com.cloudera.livy.sessions._
-
-abstract class PythonBaseInterpreterSpec extends BaseInterpreterSpec {
-
- it should "execute `1 + 2` == 3" in withInterpreter { interpreter =>
- val response = interpreter.execute("1 + 2")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "3"
- ))
- }
-
- it should "execute multiple statements" in withInterpreter { interpreter =>
- var response = interpreter.execute("x = 1")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> ""
- ))
-
- response = interpreter.execute("y = 2")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> ""
- ))
-
- response = interpreter.execute("x + y")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "3"
- ))
- }
-
- it should "execute multiple statements in one block" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """
- |x = 1
- |
- |y = 2
- |
- |x + y
- """.stripMargin)
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "3"
- ))
- }
-
- it should "parse a class" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """
- |class Counter(object):
- | def __init__(self):
- | self.count = 0
- |
- | def add_one(self):
- | self.count += 1
- |
- | def add_two(self):
- | self.count += 2
- |
- |counter = Counter()
- |counter.add_one()
- |counter.add_two()
- |counter.count
- """.stripMargin)
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "3"
- ))
- }
-
- it should "do json magic" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """x = [[1, 'a'], [3, 'b']]
- |%json x
- """.stripMargin)
-
- response should equal(Interpreter.ExecuteSuccess(
- APPLICATION_JSON -> List[JValue](
- List[JValue](1, "a"),
- List[JValue](3, "b")
- )
- ))
- }
-
- it should "do table magic" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """x = [[1, 'a'], [3, 'b']]
- |%table x
- """.stripMargin)
-
- response should equal(Interpreter.ExecuteSuccess(
- APPLICATION_LIVY_TABLE_JSON -> (
- ("headers" -> List(
- ("type" -> "INT_TYPE") ~ ("name" -> "0"),
- ("type" -> "STRING_TYPE") ~ ("name" -> "1")
- )) ~
- ("data" -> List(
- List[JValue](1, "a"),
- List[JValue](3, "b")
- ))
- )
- ))
- }
-
- it should "do table magic with None type value" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """x = [{"a":"1", "b":None}, {"a":"2", "b":2}]
- |%table x
- """.stripMargin)
-
- response should equal(Interpreter.ExecuteSuccess(
- APPLICATION_LIVY_TABLE_JSON -> (
- ("headers" -> List(
- ("type" -> "STRING_TYPE") ~ ("name" -> "a"),
- ("type" -> "INT_TYPE") ~ ("name" -> "b")
- )) ~
- ("data" -> List(
- List[JValue]("1", JNull),
- List[JValue]("2", 2)
- ))
- )
- ))
- }
-
- it should "do table magic with None type Row" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """x = [{"a":None, "b":None}, {"a":"2", "b":2}]
- |%table x
- """.stripMargin)
-
- response should equal(Interpreter.ExecuteSuccess(
- APPLICATION_LIVY_TABLE_JSON -> (
- ("headers" -> List(
- ("type" -> "STRING_TYPE") ~ ("name" -> "a"),
- ("type" -> "INT_TYPE") ~ ("name" -> "b")
- )) ~
- ("data" -> List(
- List[JValue](JNull, JNull),
- List[JValue]("2", 2)
- ))
- )
- ))
- }
-
- it should "allow magic inside statements" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """x = [[1, 'a'], [3, 'b']]
- |%table x
- |1 + 2
- """.stripMargin)
-
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "3"
- ))
- }
-
- it should "capture stdout" in withInterpreter { interpreter =>
- val response = interpreter.execute("print('Hello World')")
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "Hello World"
- ))
- }
-
- it should "report an error if accessing an unknown variable" in withInterpreter { interpreter =>
- val response = interpreter.execute("x")
- response should equal(Interpreter.ExecuteError(
- "NameError",
- "name 'x' is not defined",
- List(
- "Traceback (most recent call last):\n",
- "NameError: name 'x' is not defined\n"
- )
- ))
- }
-
- it should "report an error if empty magic command" in withInterpreter { interpreter =>
- val response = interpreter.execute("%")
- response should equal(Interpreter.ExecuteError(
- "UnknownMagic",
- "magic command not specified",
- List("UnknownMagic: magic command not specified\n")
- ))
- }
-
- it should "report an error if unknown magic command" in withInterpreter { interpreter =>
- val response = interpreter.execute("%foo")
- response should equal(Interpreter.ExecuteError(
- "UnknownMagic",
- "unknown magic command 'foo'",
- List("UnknownMagic: unknown magic command 'foo'\n")
- ))
- }
-
- it should "not execute part of the block if there is a syntax error" in withInterpreter { intp =>
- var response = intp.execute(
- """x = 1
- |'
- """.stripMargin)
-
- response should equal(Interpreter.ExecuteError(
- "SyntaxError",
- "EOL while scanning string literal (<stdin>, line 2)",
- List(
- " File \"<stdin>\", line 2\n",
- " '\n",
- " ^\n",
- "SyntaxError: EOL while scanning string literal\n"
- )
- ))
-
- response = intp.execute("x")
- response should equal(Interpreter.ExecuteError(
- "NameError",
- "name 'x' is not defined",
- List(
- "Traceback (most recent call last):\n",
- "NameError: name 'x' is not defined\n"
- )
- ))
- }
-}
-
-class Python2InterpreterSpec extends PythonBaseInterpreterSpec {
-
- implicit val formats = DefaultFormats
-
- override def createInterpreter(): Interpreter = PythonInterpreter(new SparkConf(), PySpark())
-
- // Scalastyle is treating unicode escape as non ascii characters. Turn off the check.
- // scalastyle:off non.ascii.character.disallowed
- it should "print unicode correctly" in withInterpreter { intp =>
- intp.execute("print(u\"\u263A\")") should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "\u263A"
- ))
- intp.execute("""print(u"\u263A")""") should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "\u263A"
- ))
- intp.execute("""print("\xE2\x98\xBA")""") should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "\u263A"
- ))
- }
- // scalastyle:on non.ascii.character.disallowed
-}
-
-class Python3InterpreterSpec extends PythonBaseInterpreterSpec {
-
- implicit val formats = DefaultFormats
-
- override protected def withFixture(test: NoArgTest): Outcome = {
- assume(!sys.props.getOrElse("skipPySpark3Tests", "false").toBoolean, "Skipping PySpark3 tests.")
- test()
- }
-
- override def createInterpreter(): Interpreter = PythonInterpreter(new SparkConf(), PySpark3())
-
- it should "check python version is 3.x" in withInterpreter { interpreter =>
- val response = interpreter.execute("""import sys
- |sys.version >= '3'
- """.stripMargin)
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "True"
- ))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/com/cloudera/livy/repl/PythonSessionSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/com/cloudera/livy/repl/PythonSessionSpec.scala b/repl/src/test/scala/com/cloudera/livy/repl/PythonSessionSpec.scala
deleted file mode 100644
index 6bafdf7..0000000
--- a/repl/src/test/scala/com/cloudera/livy/repl/PythonSessionSpec.scala
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import org.apache.spark.SparkConf
-import org.json4s.Extraction
-import org.json4s.jackson.JsonMethods.parse
-import org.scalatest._
-
-import com.cloudera.livy.rsc.RSCConf
-import com.cloudera.livy.sessions._
-
-abstract class PythonSessionSpec extends BaseSessionSpec {
-
- it should "execute `1 + 2` == 3" in withSession { session =>
- val statement = execute(session)("1 + 2")
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "3"
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "execute `x = 1`, then `y = 2`, then `x + y`" in withSession { session =>
- val executeWithSession = execute(session)(_)
- var statement = executeWithSession("x = 1")
- statement.id should equal (0)
-
- var result = parse(statement.output)
- var expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> ""
- )
- ))
-
- result should equal (expectedResult)
-
- statement = executeWithSession("y = 2")
- statement.id should equal (1)
-
- result = parse(statement.output)
- expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 1,
- "data" -> Map(
- "text/plain" -> ""
- )
- ))
-
- result should equal (expectedResult)
-
- statement = executeWithSession("x + y")
- statement.id should equal (2)
-
- result = parse(statement.output)
- expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 2,
- "data" -> Map(
- "text/plain" -> "3"
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "do table magic" in withSession { session =>
- val statement = execute(session)("x = [[1, 'a'], [3, 'b']]\n%table x")
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "application/vnd.livy.table.v1+json" -> Map(
- "headers" -> List(
- Map("type" -> "INT_TYPE", "name" -> "0"),
- Map("type" -> "STRING_TYPE", "name" -> "1")),
- "data" -> List(List(1, "a"), List(3, "b"))
- )
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "capture stdout" in withSession { session =>
- val statement = execute(session)("""print('Hello World')""")
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "Hello World"
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "report an error if accessing an unknown variable" in withSession { session =>
- val statement = execute(session)("""x""")
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "error",
- "execution_count" -> 0,
- "traceback" -> List(
- "Traceback (most recent call last):\n",
- "NameError: name 'x' is not defined\n"
- ),
- "ename" -> "NameError",
- "evalue" -> "name 'x' is not defined"
- ))
-
- result should equal (expectedResult)
- }
-
- it should "report an error if exception is thrown" in withSession { session =>
- val statement = execute(session)(
- """def func1():
- | raise Exception("message")
- |def func2():
- | func1()
- |func2()
- """.stripMargin)
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "error",
- "execution_count" -> 0,
- "traceback" -> List(
- "Traceback (most recent call last):\n",
- " File \"<stdin>\", line 4, in func2\n",
- " File \"<stdin>\", line 2, in func1\n",
- "Exception: message\n"
- ),
- "ename" -> "Exception",
- "evalue" -> "message"
- ))
-
- result should equal (expectedResult)
- }
-}
-
-class Python2SessionSpec extends PythonSessionSpec {
- override def createInterpreter(): Interpreter = PythonInterpreter(new SparkConf(), PySpark())
-}
-
-class Python3SessionSpec extends PythonSessionSpec {
-
- override protected def withFixture(test: NoArgTest): Outcome = {
- assume(!sys.props.getOrElse("skipPySpark3Tests", "false").toBoolean, "Skipping PySpark3 tests.")
- test()
- }
-
- override def createInterpreter(): Interpreter = PythonInterpreter(new SparkConf(), PySpark3())
-
- it should "check python version is 3.x" in withSession { session =>
- val statement = execute(session)(
- """import sys
- |sys.version >= '3'
- """.stripMargin)
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "True"
- )
- ))
-
- result should equal (expectedResult)
- }
-}
[33/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
LIVY-375. Change Livy code package name to org.apache.livy
This PR propose to change Livy code package name to org.apache.livy. This is one part to move Livy to Apache.
Author: jerryshao <ss...@hortonworks.com>
Closes #12 from jerryshao/LIVY-375.
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/412ccc8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/412ccc8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/412ccc8f
Branch: refs/heads/master
Commit: 412ccc8fcf96854fedbe76af8e5a6fec2c542d25
Parents: cb5b8aa
Author: jerryshao <ss...@hortonworks.com>
Authored: Wed Jul 5 11:06:17 2017 +0800
Committer: jerryshao <ss...@hortonworks.com>
Committed: Wed Jul 5 11:06:17 2017 +0800
----------------------------------------------------------------------
README.rst | 2 +-
api/pom.xml | 8 +-
api/src/main/java/com/cloudera/livy/Job.java | 29 -
.../main/java/com/cloudera/livy/JobContext.java | 70 ---
.../main/java/com/cloudera/livy/JobHandle.java | 76 ---
.../main/java/com/cloudera/livy/LivyClient.java | 109 ----
.../com/cloudera/livy/LivyClientBuilder.java | 163 -----
.../com/cloudera/livy/LivyClientFactory.java | 44 --
.../com/cloudera/livy/annotations/Private.java | 30 -
api/src/main/java/org/apache/livy/Job.java | 29 +
.../main/java/org/apache/livy/JobContext.java | 70 +++
.../main/java/org/apache/livy/JobHandle.java | 76 +++
.../main/java/org/apache/livy/LivyClient.java | 109 ++++
.../java/org/apache/livy/LivyClientBuilder.java | 163 +++++
.../java/org/apache/livy/LivyClientFactory.java | 44 ++
.../org/apache/livy/annotations/Private.java | 30 +
.../com/cloudera/livy/TestClientFactory.java | 86 ---
.../cloudera/livy/TestLivyClientBuilder.java | 81 ---
.../java/org/apache/livy/TestClientFactory.java | 86 +++
.../org/apache/livy/TestLivyClientBuilder.java | 81 +++
.../com.cloudera.livy.LivyClientFactory | 1 -
.../services/org.apache.livy.LivyClientFactory | 1 +
assembly/pom.xml | 6 +-
bin/livy-server | 2 +-
checkstyle.xml | 2 +-
client-common/pom.xml | 10 +-
.../livy/client/common/AbstractJobHandle.java | 105 ----
.../livy/client/common/BufferUtils.java | 42 --
.../cloudera/livy/client/common/ClientConf.java | 250 --------
.../livy/client/common/HttpMessages.java | 142 -----
.../cloudera/livy/client/common/Serializer.java | 85 ---
.../cloudera/livy/client/common/TestUtils.java | 79 ---
.../livy/client/common/AbstractJobHandle.java | 105 ++++
.../apache/livy/client/common/BufferUtils.java | 42 ++
.../apache/livy/client/common/ClientConf.java | 250 ++++++++
.../apache/livy/client/common/HttpMessages.java | 142 +++++
.../apache/livy/client/common/Serializer.java | 85 +++
.../apache/livy/client/common/TestUtils.java | 79 +++
.../client/common/TestAbstractJobHandle.java | 92 ---
.../livy/client/common/TestBufferUtils.java | 61 --
.../livy/client/common/TestClientConf.java | 230 -------
.../livy/client/common/TestHttpMessages.java | 130 ----
.../livy/client/common/TestSerializer.java | 78 ---
.../livy/client/common/TestTestUtils.java | 38 --
.../client/common/TestAbstractJobHandle.java | 92 +++
.../livy/client/common/TestBufferUtils.java | 61 ++
.../livy/client/common/TestClientConf.java | 230 +++++++
.../livy/client/common/TestHttpMessages.java | 130 ++++
.../livy/client/common/TestSerializer.java | 78 +++
.../livy/client/common/TestTestUtils.java | 38 ++
client-http/pom.xml | 28 +-
.../cloudera/livy/client/http/HttpClient.java | 193 ------
.../livy/client/http/HttpClientFactory.java | 40 --
.../com/cloudera/livy/client/http/HttpConf.java | 135 ----
.../livy/client/http/JobHandleImpl.java | 275 ---------
.../livy/client/http/LivyConnection.java | 235 -------
.../org/apache/livy/client/http/HttpClient.java | 191 ++++++
.../livy/client/http/HttpClientFactory.java | 40 ++
.../org/apache/livy/client/http/HttpConf.java | 135 ++++
.../apache/livy/client/http/JobHandleImpl.java | 273 +++++++++
.../apache/livy/client/http/LivyConnection.java | 235 +++++++
.../com.cloudera.livy.LivyClientFactory | 1 -
.../services/org.apache.livy.LivyClientFactory | 1 +
.../livy/client/http/HttpClientSpec.scala | 290 ---------
.../livy/client/http/LivyConnectionSpec.scala | 118 ----
.../livy/client/http/HttpClientSpec.scala | 290 +++++++++
.../livy/client/http/LivyConnectionSpec.scala | 118 ++++
core/pom.xml | 8 +-
core/scala-2.10/pom.xml | 8 +-
core/scala-2.11/pom.xml | 8 +-
.../main/scala/com/cloudera/livy/Logging.scala | 54 --
.../main/scala/com/cloudera/livy/Utils.scala | 109 ----
.../src/main/scala/com/cloudera/livy/msgs.scala | 63 --
.../scala/com/cloudera/livy/sessions/Kind.scala | 68 ---
.../cloudera/livy/sessions/SessionState.scala | 107 ----
.../main/scala/org/apache/livy/Logging.scala | 54 ++
core/src/main/scala/org/apache/livy/Utils.scala | 109 ++++
core/src/main/scala/org/apache/livy/msgs.scala | 63 ++
.../scala/org/apache/livy/sessions/Kind.scala | 68 +++
.../org/apache/livy/sessions/SessionState.scala | 107 ++++
.../cloudera/livy/LivyBaseUnitTestSuite.scala | 34 --
.../org/apache/livy/LivyBaseUnitTestSuite.scala | 34 ++
coverage/pom.xml | 6 +-
examples/pom.xml | 14 +-
.../java/com/cloudera/livy/examples/PiApp.java | 101 ---
.../java/org/apache/livy/examples/PiApp.java | 101 +++
.../cloudera/livy/examples/WordCountApp.scala | 218 -------
.../org/apache/livy/examples/WordCountApp.scala | 218 +++++++
.../minicluster-dependencies/pom.xml | 6 +-
.../minicluster-dependencies/scala-2.10/pom.xml | 8 +-
.../minicluster-dependencies/scala-2.11/pom.xml | 8 +-
integration-test/pom.xml | 6 +-
.../framework/BaseIntegrationTestSuite.scala | 120 ----
.../cloudera/livy/test/framework/Cluster.scala | 160 -----
.../livy/test/framework/LivyRestClient.scala | 255 --------
.../livy/test/framework/MiniCluster.scala | 386 ------------
.../livy/test/framework/RealCluster.scala | 277 ---------
.../framework/BaseIntegrationTestSuite.scala | 120 ++++
.../apache/livy/test/framework/Cluster.scala | 160 +++++
.../livy/test/framework/LivyRestClient.scala | 255 ++++++++
.../livy/test/framework/MiniCluster.scala | 386 ++++++++++++
.../livy/test/framework/RealCluster.scala | 277 +++++++++
.../scala/com/cloudera/livy/test/BatchIT.scala | 174 ------
.../com/cloudera/livy/test/InteractiveIT.scala | 205 -------
.../scala/com/cloudera/livy/test/JobApiIT.scala | 294 ---------
.../scala/org/apache/livy/test/BatchIT.scala | 174 ++++++
.../org/apache/livy/test/InteractiveIT.scala | 205 +++++++
.../scala/org/apache/livy/test/JobApiIT.scala | 294 +++++++++
.../src/test/spark2/scala/Spark2JobApiIT.scala | 12 +-
pom.xml | 4 +-
python-api/pom.xml | 8 +-
python-api/setup.py | 6 +-
repl/pom.xml | 14 +-
repl/scala-2.10/pom.xml | 8 +-
.../cloudera/livy/repl/SparkInterpreter.scala | 178 ------
.../org/apache/livy/repl/SparkInterpreter.scala | 178 ++++++
.../livy/repl/SparkInterpreterSpec.scala | 86 ---
.../apache/livy/repl/SparkInterpreterSpec.scala | 86 +++
repl/scala-2.11/pom.xml | 8 +-
.../cloudera/livy/repl/SparkInterpreter.scala | 174 ------
.../org/apache/livy/repl/SparkInterpreter.scala | 174 ++++++
.../livy/repl/SparkInterpreterSpec.scala | 68 ---
.../apache/livy/repl/SparkInterpreterSpec.scala | 68 +++
repl/src/main/resources/fake_shell.py | 2 +-
.../livy/repl/AbstractSparkInterpreter.scala | 268 --------
.../cloudera/livy/repl/BypassPySparkJob.scala | 40 --
.../com/cloudera/livy/repl/Interpreter.scala | 54 --
.../cloudera/livy/repl/ProcessInterpreter.scala | 137 -----
.../livy/repl/PySparkJobProcessor.scala | 27 -
.../cloudera/livy/repl/PythonInterpreter.scala | 293 ---------
.../com/cloudera/livy/repl/ReplDriver.scala | 131 ----
.../scala/com/cloudera/livy/repl/Session.scala | 289 ---------
.../livy/repl/SparkContextInitializer.scala | 124 ----
.../cloudera/livy/repl/SparkRInterpreter.scala | 324 ----------
.../scala/com/cloudera/livy/repl/package.scala | 29 -
.../livy/repl/AbstractSparkInterpreter.scala | 268 ++++++++
.../org/apache/livy/repl/BypassPySparkJob.scala | 40 ++
.../org/apache/livy/repl/Interpreter.scala | 54 ++
.../apache/livy/repl/ProcessInterpreter.scala | 137 +++++
.../apache/livy/repl/PySparkJobProcessor.scala | 27 +
.../apache/livy/repl/PythonInterpreter.scala | 293 +++++++++
.../scala/org/apache/livy/repl/ReplDriver.scala | 131 ++++
.../scala/org/apache/livy/repl/Session.scala | 289 +++++++++
.../livy/repl/SparkContextInitializer.scala | 124 ++++
.../apache/livy/repl/SparkRInterpreter.scala | 324 ++++++++++
.../scala/org/apache/livy/repl/package.scala | 29 +
.../livy/repl/BaseInterpreterSpec.scala | 37 --
.../cloudera/livy/repl/BaseSessionSpec.scala | 87 ---
.../livy/repl/PythonInterpreterSpec.scala | 284 ---------
.../cloudera/livy/repl/PythonSessionSpec.scala | 206 -------
.../cloudera/livy/repl/ReplDriverSuite.scala | 68 ---
.../livy/repl/ScalaInterpreterSpec.scala | 206 -------
.../com/cloudera/livy/repl/SessionSpec.scala | 126 ----
.../livy/repl/SparkRInterpreterSpec.scala | 107 ----
.../cloudera/livy/repl/SparkRSessionSpec.scala | 140 -----
.../cloudera/livy/repl/SparkSessionSpec.scala | 273 ---------
.../apache/livy/repl/BaseInterpreterSpec.scala | 37 ++
.../org/apache/livy/repl/BaseSessionSpec.scala | 87 +++
.../livy/repl/PythonInterpreterSpec.scala | 284 +++++++++
.../apache/livy/repl/PythonSessionSpec.scala | 206 +++++++
.../org/apache/livy/repl/ReplDriverSuite.scala | 68 +++
.../apache/livy/repl/ScalaInterpreterSpec.scala | 206 +++++++
.../org/apache/livy/repl/SessionSpec.scala | 126 ++++
.../livy/repl/SparkRInterpreterSpec.scala | 107 ++++
.../apache/livy/repl/SparkRSessionSpec.scala | 140 +++++
.../org/apache/livy/repl/SparkSessionSpec.scala | 273 +++++++++
rsc/pom.xml | 18 +-
.../com/cloudera/livy/rsc/BaseProtocol.java | 241 --------
.../com/cloudera/livy/rsc/BypassJobStatus.java | 40 --
.../java/com/cloudera/livy/rsc/ContextInfo.java | 37 --
.../com/cloudera/livy/rsc/ContextLauncher.java | 456 --------------
.../cloudera/livy/rsc/DriverProcessInfo.java | 42 --
.../com/cloudera/livy/rsc/FutureListener.java | 27 -
.../com/cloudera/livy/rsc/JobHandleImpl.java | 106 ----
.../java/com/cloudera/livy/rsc/PingJob.java | 31 -
.../java/com/cloudera/livy/rsc/RSCClient.java | 409 -------------
.../com/cloudera/livy/rsc/RSCClientFactory.java | 116 ----
.../java/com/cloudera/livy/rsc/RSCConf.java | 212 -------
.../com/cloudera/livy/rsc/ReplJobResults.java | 31 -
.../main/java/com/cloudera/livy/rsc/Utils.java | 118 ----
.../cloudera/livy/rsc/driver/AddFileJob.java | 40 --
.../com/cloudera/livy/rsc/driver/AddJarJob.java | 42 --
.../com/cloudera/livy/rsc/driver/BypassJob.java | 51 --
.../livy/rsc/driver/BypassJobWrapper.java | 75 ---
.../livy/rsc/driver/JobContextImpl.java | 147 -----
.../cloudera/livy/rsc/driver/JobWrapper.java | 99 ---
.../livy/rsc/driver/MutableClassLoader.java | 34 --
.../com/cloudera/livy/rsc/driver/RSCDriver.java | 510 ----------------
.../livy/rsc/driver/RSCDriverBootstrapper.java | 89 ---
.../com/cloudera/livy/rsc/driver/Statement.java | 59 --
.../livy/rsc/driver/StatementState.java | 86 ---
.../cloudera/livy/rsc/rpc/KryoMessageCodec.java | 162 -----
.../java/com/cloudera/livy/rsc/rpc/README.md | 32 -
.../java/com/cloudera/livy/rsc/rpc/Rpc.java | 460 --------------
.../cloudera/livy/rsc/rpc/RpcDispatcher.java | 219 -------
.../com/cloudera/livy/rsc/rpc/RpcException.java | 26 -
.../com/cloudera/livy/rsc/rpc/RpcServer.java | 368 -----------
.../com/cloudera/livy/rsc/rpc/SaslHandler.java | 116 ----
.../java/org/apache/livy/rsc/BaseProtocol.java | 240 ++++++++
.../org/apache/livy/rsc/BypassJobStatus.java | 38 ++
.../java/org/apache/livy/rsc/ContextInfo.java | 37 ++
.../org/apache/livy/rsc/ContextLauncher.java | 457 ++++++++++++++
.../org/apache/livy/rsc/DriverProcessInfo.java | 42 ++
.../org/apache/livy/rsc/FutureListener.java | 27 +
.../java/org/apache/livy/rsc/JobHandleImpl.java | 106 ++++
.../main/java/org/apache/livy/rsc/PingJob.java | 31 +
.../java/org/apache/livy/rsc/RSCClient.java | 410 +++++++++++++
.../org/apache/livy/rsc/RSCClientFactory.java | 116 ++++
.../main/java/org/apache/livy/rsc/RSCConf.java | 212 +++++++
.../org/apache/livy/rsc/ReplJobResults.java | 31 +
.../main/java/org/apache/livy/rsc/Utils.java | 118 ++++
.../org/apache/livy/rsc/driver/AddFileJob.java | 40 ++
.../org/apache/livy/rsc/driver/AddJarJob.java | 42 ++
.../org/apache/livy/rsc/driver/BypassJob.java | 51 ++
.../livy/rsc/driver/BypassJobWrapper.java | 75 +++
.../apache/livy/rsc/driver/JobContextImpl.java | 147 +++++
.../org/apache/livy/rsc/driver/JobWrapper.java | 99 +++
.../livy/rsc/driver/MutableClassLoader.java | 34 ++
.../org/apache/livy/rsc/driver/RSCDriver.java | 510 ++++++++++++++++
.../livy/rsc/driver/RSCDriverBootstrapper.java | 89 +++
.../org/apache/livy/rsc/driver/Statement.java | 59 ++
.../apache/livy/rsc/driver/StatementState.java | 86 +++
.../apache/livy/rsc/rpc/KryoMessageCodec.java | 162 +++++
.../main/java/org/apache/livy/rsc/rpc/README.md | 32 +
.../main/java/org/apache/livy/rsc/rpc/Rpc.java | 460 ++++++++++++++
.../org/apache/livy/rsc/rpc/RpcDispatcher.java | 219 +++++++
.../org/apache/livy/rsc/rpc/RpcException.java | 26 +
.../java/org/apache/livy/rsc/rpc/RpcServer.java | 368 +++++++++++
.../org/apache/livy/rsc/rpc/SaslHandler.java | 116 ++++
.../com.cloudera.livy.LivyClientFactory | 1 -
.../services/org.apache.livy.LivyClientFactory | 1 +
.../com/cloudera/livy/rsc/TestJobHandle.java | 98 ---
.../com/cloudera/livy/rsc/TestSparkClient.java | 533 ----------------
.../livy/rsc/rpc/TestKryoMessageCodec.java | 232 -------
.../java/com/cloudera/livy/rsc/rpc/TestRpc.java | 337 ----------
.../java/org/apache/livy/rsc/TestJobHandle.java | 98 +++
.../org/apache/livy/rsc/TestSparkClient.java | 533 ++++++++++++++++
.../livy/rsc/rpc/TestKryoMessageCodec.java | 232 +++++++
.../java/org/apache/livy/rsc/rpc/TestRpc.java | 337 ++++++++++
scala-api/pom.xml | 14 +-
scala-api/scala-2.10/pom.xml | 8 +-
scala-api/scala-2.11/pom.xml | 8 +-
.../livy/scalaapi/LivyScalaClient.scala | 165 -----
.../livy/scalaapi/ScalaJobContext.scala | 67 --
.../cloudera/livy/scalaapi/ScalaJobHandle.scala | 205 -------
.../com/cloudera/livy/scalaapi/package.scala | 50 --
.../apache/livy/scalaapi/LivyScalaClient.scala | 165 +++++
.../apache/livy/scalaapi/ScalaJobContext.scala | 67 ++
.../apache/livy/scalaapi/ScalaJobHandle.scala | 205 +++++++
.../org/apache/livy/scalaapi/package.scala | 50 ++
.../livy/scalaapi/ScalaClientTest.scala | 216 -------
.../livy/scalaapi/ScalaClientTestUtils.scala | 57 --
.../livy/scalaapi/ScalaJobHandleTest.scala | 190 ------
.../apache/livy/scalaapi/ScalaClientTest.scala | 216 +++++++
.../livy/scalaapi/ScalaClientTestUtils.scala | 57 ++
.../livy/scalaapi/ScalaJobHandleTest.scala | 190 ++++++
scala/pom.xml | 8 +-
scalastyle.xml | 4 +-
server/pom.xml | 12 +-
.../livy/server/ui/static/all-sessions.js | 93 ---
.../livy/server/ui/static/batches-table.html | 42 --
.../livy/server/ui/static/bootstrap.min.css | 14 -
.../livy/server/ui/static/bootstrap.min.js | 7 -
.../ui/static/dataTables.bootstrap.min.css | 1 -
.../ui/static/dataTables.bootstrap.min.js | 8 -
.../livy/server/ui/static/jquery-3.2.1.min.js | 4 -
.../server/ui/static/jquery.dataTables.min.js | 167 -----
.../livy/server/ui/static/livy-mini-logo.png | Bin 1121 -> 0 bytes
.../cloudera/livy/server/ui/static/livy-ui.css | 20 -
.../livy/server/ui/static/sessions-table.html | 59 --
.../livy/server/ui/static/all-sessions.js | 93 +++
.../livy/server/ui/static/batches-table.html | 42 ++
.../livy/server/ui/static/bootstrap.min.css | 14 +
.../livy/server/ui/static/bootstrap.min.js | 7 +
.../ui/static/dataTables.bootstrap.min.css | 1 +
.../ui/static/dataTables.bootstrap.min.js | 8 +
.../livy/server/ui/static/jquery-3.2.1.min.js | 4 +
.../server/ui/static/jquery.dataTables.min.js | 167 +++++
.../livy/server/ui/static/livy-mini-logo.png | Bin 0 -> 1121 bytes
.../apache/livy/server/ui/static/livy-ui.css | 20 +
.../livy/server/ui/static/sessions-table.html | 59 ++
.../main/scala/com/cloudera/livy/LivyConf.scala | 297 ---------
.../main/scala/com/cloudera/livy/package.scala | 73 ---
.../com/cloudera/livy/server/AccessFilter.scala | 45 --
.../livy/server/ApiVersioningSupport.scala | 93 ---
.../com/cloudera/livy/server/ApiVersions.scala | 35 --
.../com/cloudera/livy/server/CsrfFilter.scala | 49 --
.../com/cloudera/livy/server/JsonServlet.scala | 143 -----
.../com/cloudera/livy/server/LivyServer.scala | 348 -----------
.../cloudera/livy/server/SessionServlet.scala | 215 -------
.../com/cloudera/livy/server/WebServer.scala | 113 ----
.../livy/server/batch/BatchSession.scala | 173 ------
.../livy/server/batch/BatchSessionServlet.scala | 67 --
.../livy/server/batch/CreateBatchRequest.scala | 56 --
.../interactive/CreateInteractiveRequest.scala | 56 --
.../server/interactive/InteractiveSession.scala | 609 -------------------
.../interactive/InteractiveSessionServlet.scala | 247 --------
.../server/interactive/SessionHeartbeat.scala | 113 ----
.../server/recovery/BlackholeStateStore.scala | 36 --
.../server/recovery/FileSystemStateStore.scala | 132 ----
.../livy/server/recovery/SessionStore.scala | 96 ---
.../livy/server/recovery/StateStore.scala | 111 ----
.../server/recovery/ZooKeeperStateStore.scala | 118 ----
.../com/cloudera/livy/server/ui/UIServlet.scala | 78 ---
.../com/cloudera/livy/sessions/Session.scala | 264 --------
.../cloudera/livy/sessions/SessionManager.scala | 188 ------
.../scala/com/cloudera/livy/utils/Clock.scala | 38 --
.../livy/utils/LineBufferedProcess.scala | 51 --
.../livy/utils/LineBufferedStream.scala | 97 ---
.../cloudera/livy/utils/LivySparkUtils.scala | 197 ------
.../com/cloudera/livy/utils/SparkApp.scala | 106 ----
.../com/cloudera/livy/utils/SparkProcApp.scala | 60 --
.../livy/utils/SparkProcessBuilder.scala | 219 -------
.../com/cloudera/livy/utils/SparkYarnApp.scala | 312 ----------
.../main/scala/org/apache/livy/LivyConf.scala | 297 +++++++++
.../main/scala/org/apache/livy/package.scala | 73 +++
.../org/apache/livy/server/AccessFilter.scala | 45 ++
.../livy/server/ApiVersioningSupport.scala | 93 +++
.../org/apache/livy/server/ApiVersions.scala | 35 ++
.../org/apache/livy/server/CsrfFilter.scala | 48 ++
.../org/apache/livy/server/JsonServlet.scala | 143 +++++
.../org/apache/livy/server/LivyServer.scala | 348 +++++++++++
.../org/apache/livy/server/SessionServlet.scala | 215 +++++++
.../org/apache/livy/server/WebServer.scala | 113 ++++
.../apache/livy/server/batch/BatchSession.scala | 173 ++++++
.../livy/server/batch/BatchSessionServlet.scala | 67 ++
.../livy/server/batch/CreateBatchRequest.scala | 56 ++
.../interactive/CreateInteractiveRequest.scala | 56 ++
.../server/interactive/InteractiveSession.scala | 608 ++++++++++++++++++
.../interactive/InteractiveSessionServlet.scala | 247 ++++++++
.../server/interactive/SessionHeartbeat.scala | 113 ++++
.../server/recovery/BlackholeStateStore.scala | 36 ++
.../server/recovery/FileSystemStateStore.scala | 132 ++++
.../livy/server/recovery/SessionStore.scala | 96 +++
.../livy/server/recovery/StateStore.scala | 111 ++++
.../server/recovery/ZooKeeperStateStore.scala | 118 ++++
.../org/apache/livy/server/ui/UIServlet.scala | 76 +++
.../org/apache/livy/sessions/Session.scala | 264 ++++++++
.../apache/livy/sessions/SessionManager.scala | 188 ++++++
.../scala/org/apache/livy/utils/Clock.scala | 38 ++
.../apache/livy/utils/LineBufferedProcess.scala | 51 ++
.../apache/livy/utils/LineBufferedStream.scala | 97 +++
.../org/apache/livy/utils/LivySparkUtils.scala | 196 ++++++
.../scala/org/apache/livy/utils/SparkApp.scala | 105 ++++
.../org/apache/livy/utils/SparkProcApp.scala | 59 ++
.../apache/livy/utils/SparkProcessBuilder.scala | 218 +++++++
.../org/apache/livy/utils/SparkYarnApp.scala | 311 ++++++++++
.../livy/server/ApiVersioningSupportSpec.scala | 124 ----
.../livy/server/BaseJsonServletSpec.scala | 141 -----
.../livy/server/BaseSessionServletSpec.scala | 82 ---
.../cloudera/livy/server/JsonServletSpec.scala | 149 -----
.../livy/server/SessionServletSpec.scala | 156 -----
.../livy/server/batch/BatchServletSpec.scala | 149 -----
.../livy/server/batch/BatchSessionSpec.scala | 113 ----
.../server/batch/CreateBatchRequestSpec.scala | 55 --
.../BaseInteractiveServletSpec.scala | 74 ---
.../CreateInteractiveRequestSpec.scala | 55 --
.../InteractiveSessionServletSpec.scala | 183 ------
.../interactive/InteractiveSessionSpec.scala | 264 --------
.../livy/server/interactive/JobApiSpec.scala | 227 -------
.../interactive/SessionHeartbeatSpec.scala | 87 ---
.../recovery/BlackholeStateStoreSpec.scala | 47 --
.../recovery/FileSystemStateStoreSpec.scala | 192 ------
.../livy/server/recovery/SessionStoreSpec.scala | 108 ----
.../livy/server/recovery/StateStoreSpec.scala | 65 --
.../recovery/ZooKeeperStateStoreSpec.scala | 174 ------
.../cloudera/livy/sessions/MockSession.scala | 34 --
.../livy/sessions/SessionManagerSpec.scala | 205 -------
.../cloudera/livy/sessions/SessionSpec.scala | 98 ---
.../livy/utils/LivySparkUtilsSuite.scala | 140 -----
.../cloudera/livy/utils/SparkYarnAppSpec.scala | 353 -----------
.../livy/server/ApiVersioningSupportSpec.scala | 124 ++++
.../livy/server/BaseJsonServletSpec.scala | 141 +++++
.../livy/server/BaseSessionServletSpec.scala | 82 +++
.../apache/livy/server/JsonServletSpec.scala | 149 +++++
.../apache/livy/server/SessionServletSpec.scala | 155 +++++
.../livy/server/batch/BatchServletSpec.scala | 149 +++++
.../livy/server/batch/BatchSessionSpec.scala | 113 ++++
.../server/batch/CreateBatchRequestSpec.scala | 55 ++
.../BaseInteractiveServletSpec.scala | 74 +++
.../CreateInteractiveRequestSpec.scala | 55 ++
.../InteractiveSessionServletSpec.scala | 183 ++++++
.../interactive/InteractiveSessionSpec.scala | 264 ++++++++
.../livy/server/interactive/JobApiSpec.scala | 227 +++++++
.../interactive/SessionHeartbeatSpec.scala | 87 +++
.../recovery/BlackholeStateStoreSpec.scala | 47 ++
.../recovery/FileSystemStateStoreSpec.scala | 192 ++++++
.../livy/server/recovery/SessionStoreSpec.scala | 108 ++++
.../livy/server/recovery/StateStoreSpec.scala | 65 ++
.../recovery/ZooKeeperStateStoreSpec.scala | 174 ++++++
.../org/apache/livy/sessions/MockSession.scala | 34 ++
.../livy/sessions/SessionManagerSpec.scala | 205 +++++++
.../org/apache/livy/sessions/SessionSpec.scala | 98 +++
.../apache/livy/utils/LivySparkUtilsSuite.scala | 140 +++++
.../apache/livy/utils/SparkYarnAppSpec.scala | 352 +++++++++++
test-lib/pom.xml | 10 +-
.../com/cloudera/livy/test/apps/FailingApp.java | 39 --
.../cloudera/livy/test/apps/SimpleSparkApp.java | 71 ---
.../java/com/cloudera/livy/test/jobs/Echo.java | 36 --
.../com/cloudera/livy/test/jobs/Failure.java | 34 --
.../com/cloudera/livy/test/jobs/FileReader.java | 79 ---
.../cloudera/livy/test/jobs/GetCurrentUser.java | 32 -
.../cloudera/livy/test/jobs/SQLGetTweets.java | 76 ---
.../com/cloudera/livy/test/jobs/Sleeper.java | 37 --
.../com/cloudera/livy/test/jobs/SmallCount.java | 48 --
.../com/cloudera/livy/test/jobs/VoidJob.java | 28 -
.../org/apache/livy/test/apps/FailingApp.java | 39 ++
.../apache/livy/test/apps/SimpleSparkApp.java | 71 +++
.../java/org/apache/livy/test/jobs/Echo.java | 36 ++
.../java/org/apache/livy/test/jobs/Failure.java | 34 ++
.../org/apache/livy/test/jobs/FileReader.java | 79 +++
.../apache/livy/test/jobs/GetCurrentUser.java | 32 +
.../org/apache/livy/test/jobs/SQLGetTweets.java | 76 +++
.../java/org/apache/livy/test/jobs/Sleeper.java | 37 ++
.../org/apache/livy/test/jobs/SmallCount.java | 48 ++
.../java/org/apache/livy/test/jobs/VoidJob.java | 28 +
.../com/cloudera/livy/test/jobs/ScalaEcho.scala | 32 -
.../org/apache/livy/test/jobs/ScalaEcho.scala | 32 +
.../livy/test/jobs/spark2/DatasetTest.java | 63 --
.../livy/test/jobs/spark2/SparkSessionTest.java | 38 --
.../livy/test/jobs/spark2/DatasetTest.java | 63 ++
.../livy/test/jobs/spark2/SparkSessionTest.java | 38 ++
422 files changed, 25580 insertions(+), 25596 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/README.rst
----------------------------------------------------------------------
diff --git a/README.rst b/README.rst
index d72c955..9f14fa0 100644
--- a/README.rst
+++ b/README.rst
@@ -190,7 +190,7 @@ And add the Livy client dependency:
To be able to compile code that uses Spark APIs, also add the correspondent Spark dependencies.
-To run Spark jobs within your applications, extend ``com.cloudera.livy.Job`` and implement
+To run Spark jobs within your applications, extend ``org.apache.livy.Job`` and implement
the functionality you need. Here's an example job that calculates an approximate value for Pi:
.. code:: java
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/pom.xml
----------------------------------------------------------------------
diff --git a/api/pom.xml b/api/pom.xml
index 156d987..0b1411e 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -18,14 +18,14 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
</parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-api</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/Job.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/cloudera/livy/Job.java b/api/src/main/java/com/cloudera/livy/Job.java
deleted file mode 100644
index fc96b72..0000000
--- a/api/src/main/java/com/cloudera/livy/Job.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy;
-
-import java.io.Serializable;
-
-/**
- * Interface for a Spark remote job.
- */
-public interface Job<T> extends Serializable {
-
- T call(JobContext jc) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/JobContext.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/cloudera/livy/JobContext.java b/api/src/main/java/com/cloudera/livy/JobContext.java
deleted file mode 100644
index d32adef..0000000
--- a/api/src/main/java/com/cloudera/livy/JobContext.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy;
-
-import java.io.File;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
-import org.apache.spark.sql.hive.HiveContext;
-import org.apache.spark.streaming.api.java.JavaStreamingContext;
-
-/**
- * Holds runtime information about the job execution context.
- *
- * An instance of this class is kept on the node hosting a remote Spark context and is made
- * available to jobs being executed via RemoteSparkContext#submit().
- */
-public interface JobContext {
-
- /** The shared SparkContext instance. */
- JavaSparkContext sc();
-
- /** The shared SQLContext instance. */
- SQLContext sqlctx();
-
- /** The shared HiveContext instance. */
- HiveContext hivectx();
-
- /** Returns the JavaStreamingContext which has already been created. */
- JavaStreamingContext streamingctx();
-
- /**
- * Creates the SparkStreaming context.
- *
- * @param batchDuration Time interval at which streaming data will be divided into batches,
- * in milliseconds.
- */
- void createStreamingContext(long batchDuration);
-
- /** Stops the SparkStreaming context. */
- void stopStreamingCtx();
-
- /**
- * Returns a local tmp dir specific to the context
- */
- File getLocalTmpDir();
-
- /**
- * Returns SparkSession if it existed, otherwise throws Exception.
- */
- <E> E sparkSession() throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/JobHandle.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/cloudera/livy/JobHandle.java b/api/src/main/java/com/cloudera/livy/JobHandle.java
deleted file mode 100644
index 1da606e..0000000
--- a/api/src/main/java/com/cloudera/livy/JobHandle.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy;
-
-import java.util.List;
-import java.util.concurrent.Future;
-
-/**
- * A handle to a submitted job. Allows for monitoring and controlling of the running remote job.
- */
-public interface JobHandle<T> extends Future<T> {
-
- /**
- * Return the current state of the job.
- */
- State getState();
-
- /**
- * Add a listener to the job handle. If the job's state is not SENT, a callback for the
- * corresponding state will be invoked immediately.
- *
- * @param l The listener to add.
- */
- void addListener(Listener<T> l);
-
- /**
- * The current state of the submitted job.
- */
- static enum State {
- SENT,
- QUEUED,
- STARTED,
- CANCELLED,
- FAILED,
- SUCCEEDED;
- }
-
- /**
- * A listener for monitoring the state of the job in the remote context. Callbacks are called
- * when the corresponding state change occurs.
- */
- static interface Listener<T> {
-
- /**
- * Notifies when a job has been queued for execution on the remote context. Note that it is
- * possible for jobs to bypass this state and got directly from the SENT state to the STARTED
- * state.
- */
- void onJobQueued(JobHandle<T> job);
-
- void onJobStarted(JobHandle<T> job);
-
- void onJobCancelled(JobHandle<T> job);
-
- void onJobFailed(JobHandle<T> job, Throwable cause);
-
- void onJobSucceeded(JobHandle<T> job, T result);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/LivyClient.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/cloudera/livy/LivyClient.java b/api/src/main/java/com/cloudera/livy/LivyClient.java
deleted file mode 100644
index 73e318f..0000000
--- a/api/src/main/java/com/cloudera/livy/LivyClient.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy;
-
-import java.io.File;
-import java.net.URI;
-import java.util.concurrent.Future;
-
-/**
- * A client for submitting Spark-based jobs to a Livy backend.
- */
-public interface LivyClient {
-
- /**
- * Submits a job for asynchronous execution.
- *
- * @param job The job to execute.
- * @return A handle that be used to monitor the job.
- */
- <T> JobHandle<T> submit(Job<T> job);
-
- /**
- * Asks the remote context to run a job immediately.
- * <p/>
- * Normally, the remote context will queue jobs and execute them based on how many worker
- * threads have been configured. This method will run the submitted job in the same thread
- * processing the RPC message, so that queueing does not apply.
- * <p/>
- * It's recommended that this method only be used to run code that finishes quickly. This
- * avoids interfering with the normal operation of the context.
- * <p/>
- * Note: the {@link JobContext#monitor()} functionality is not available when using this method.
- *
- * @param job The job to execute.
- * @return A future to monitor the result of the job.
- */
- <T> Future<T> run(Job<T> job);
-
- /**
- * Stops the remote context.
- *
- * Any pending jobs will be cancelled, and the remote context will be torn down.
- *
- * @param shutdownContext Whether to shutdown the underlying Spark context. If false, the
- * context will keep running and it's still possible to send commands
- * to it, if the backend being used supports it.
- */
- void stop(boolean shutdownContext);
-
- /**
- * Upload a jar to be added to the Spark application classpath
- * @param jar The local file to be uploaded
- * @return A future that can be used to monitor this operation
- */
- Future<?> uploadJar(File jar);
-
- /**
- * Adds a jar file to the running remote context.
- * <p>
- * Note that the URL should be reachable by the Spark driver process. If running the driver
- * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
- * on that node (and not on the client machine).
- * <p>
- * If the provided URI has no scheme, it's considered to be relative to the default file system
- * configured in the Livy server.
- *
- * @param uri The location of the jar file.
- * @return A future that can be used to monitor the operation.
- */
- Future<?> addJar(URI uri);
-
- /**
- * Upload a file to be passed to the Spark application
- * @param file The local file to be uploaded
- * @return A future that can be used to monitor this operation
- */
- Future<?> uploadFile(File file);
-
- /**
- * Adds a file to the running remote context.
- * <p>
- * Note that the URL should be reachable by the Spark driver process. If running the driver
- * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
- * on that node (and not on the client machine).
- * <p>
- * If the provided URI has no scheme, it's considered to be relative to the default file system
- * configured in the Livy server.
- *
- * @param uri The location of the file.
- * @return A future that can be used to monitor the operation.
- */
- Future<?> addFile(URI uri);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/LivyClientBuilder.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/cloudera/livy/LivyClientBuilder.java b/api/src/main/java/com/cloudera/livy/LivyClientBuilder.java
deleted file mode 100644
index 5203f04..0000000
--- a/api/src/main/java/com/cloudera/livy/LivyClientBuilder.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy;
-
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.Map;
-import java.util.Properties;
-import java.util.ServiceLoader;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-/**
- * A builder for Livy clients.
- */
-public final class LivyClientBuilder {
-
- public static final String LIVY_URI_KEY = "livy.uri";
-
- private final Properties config;
-
- /**
- * Creates a new builder that will automatically load the default Livy and Spark configuration
- * from the classpath.
- */
- public LivyClientBuilder() throws IOException {
- this(true);
- }
-
- /**
- * Creates a new builder that will optionally load the default Livy and Spark configuration
- * from the classpath.
- *
- * Livy client configuration is stored in a file called "livy-client.conf", and Spark client
- * configuration is stored in a file called "spark-defaults.conf", both in the root of the
- * application's classpath. Livy configuration takes precedence over Spark's (in case
- * configuration entries are duplicated), and configuration set in this builder object will
- * override the values in those files.
- */
- public LivyClientBuilder(boolean loadDefaults) throws IOException {
- this.config = new Properties();
-
- if (loadDefaults) {
- String[] confFiles = { "spark-defaults.conf", "livy-client.conf" };
-
- for (String file : confFiles) {
- URL url = classLoader().getResource(file);
- if (url != null) {
- Reader r = new InputStreamReader(url.openStream(), UTF_8);
- try {
- config.load(r);
- } finally {
- r.close();
- }
- }
- }
- }
- }
-
- public LivyClientBuilder setURI(URI uri) {
- config.setProperty(LIVY_URI_KEY, uri.toString());
- return this;
- }
-
- public LivyClientBuilder setConf(String key, String value) {
- if (value != null) {
- config.setProperty(key, value);
- } else {
- config.remove(key);
- }
- return this;
- }
-
- public LivyClientBuilder setAll(Map<String, String> props) {
- config.putAll(props);
- return this;
- }
-
- public LivyClientBuilder setAll(Properties props) {
- config.putAll(props);
- return this;
- }
-
- public LivyClient build() {
- String uriStr = config.getProperty(LIVY_URI_KEY);
- if (uriStr == null) {
- throw new IllegalArgumentException("URI must be provided.");
- }
- URI uri;
- try {
- uri = new URI(uriStr);
- } catch (URISyntaxException e) {
- throw new IllegalArgumentException("Invalid URI.", e);
- }
-
- LivyClient client = null;
- ServiceLoader<LivyClientFactory> loader = ServiceLoader.load(LivyClientFactory.class,
- classLoader());
- if (!loader.iterator().hasNext()) {
- throw new IllegalStateException("No LivyClientFactory implementation was found.");
- }
-
- Exception error = null;
- for (LivyClientFactory factory : loader) {
- try {
- client = factory.createClient(uri, config);
- } catch (Exception e) {
- if (!(e instanceof RuntimeException)) {
- e = new RuntimeException(e);
- }
- throw (RuntimeException) e;
- }
- if (client != null) {
- break;
- }
- }
-
- if (client == null) {
- // Redact any user information from the URI when throwing user-visible exceptions that might
- // be logged.
- if (uri.getUserInfo() != null) {
- try {
- uri = new URI(uri.getScheme(), "[redacted]", uri.getHost(), uri.getPort(), uri.getPath(),
- uri.getQuery(), uri.getFragment());
- } catch (URISyntaxException e) {
- // Shouldn't really happen.
- throw new RuntimeException(e);
- }
- }
-
- throw new IllegalArgumentException(String.format(
- "URI '%s' is not supported by any registered client factories.", uri));
- }
- return client;
- }
-
- private ClassLoader classLoader() {
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
- if (cl == null) {
- cl = getClass().getClassLoader();
- }
- return cl;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/LivyClientFactory.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/cloudera/livy/LivyClientFactory.java b/api/src/main/java/com/cloudera/livy/LivyClientFactory.java
deleted file mode 100644
index 095d5e4..0000000
--- a/api/src/main/java/com/cloudera/livy/LivyClientFactory.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy;
-
-import java.net.URI;
-import java.util.Properties;
-
-import com.cloudera.livy.annotations.Private;
-
-/**
- * A factory for Livy clients. Client implementations can register themselves by using the
- * Java services mechanism, providing implementations of this interface.
- * <p>
- * Client applications do not need to use this interface directly. Instead, use
- * {@link LivyClientBuilder}.
- *
- * @see java.util.ServiceLoader
- */
-@Private
-public interface LivyClientFactory {
-
- /**
- * Instantiates a new client if the given URI is supported by the implementation.
- *
- * @param uri URI pointing at the livy backend to use.
- */
- LivyClient createClient(URI uri, Properties config);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/annotations/Private.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/cloudera/livy/annotations/Private.java b/api/src/main/java/com/cloudera/livy/annotations/Private.java
deleted file mode 100644
index 4fc62f7..0000000
--- a/api/src/main/java/com/cloudera/livy/annotations/Private.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.annotations;
-
-import java.lang.annotation.Documented;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-
-/**
- * Indicates an API that is considered private to Livy and should not be used by client
- * applications.
- */
-@Documented
-@Retention(RetentionPolicy.CLASS)
-public @interface Private { }
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/Job.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/livy/Job.java b/api/src/main/java/org/apache/livy/Job.java
new file mode 100644
index 0000000..cbb6b9a
--- /dev/null
+++ b/api/src/main/java/org/apache/livy/Job.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy;
+
+import java.io.Serializable;
+
+/**
+ * Interface for a Spark remote job.
+ */
+public interface Job<T> extends Serializable {
+
+ T call(JobContext jc) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/JobContext.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/livy/JobContext.java b/api/src/main/java/org/apache/livy/JobContext.java
new file mode 100644
index 0000000..47027b7
--- /dev/null
+++ b/api/src/main/java/org/apache/livy/JobContext.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.hive.HiveContext;
+import org.apache.spark.streaming.api.java.JavaStreamingContext;
+
+/**
+ * Holds runtime information about the job execution context.
+ *
+ * An instance of this class is kept on the node hosting a remote Spark context and is made
+ * available to jobs being executed via RemoteSparkContext#submit().
+ */
+public interface JobContext {
+
+ /** The shared SparkContext instance. */
+ JavaSparkContext sc();
+
+ /** The shared SQLContext instance. */
+ SQLContext sqlctx();
+
+ /** The shared HiveContext instance. */
+ HiveContext hivectx();
+
+ /** Returns the JavaStreamingContext which has already been created. */
+ JavaStreamingContext streamingctx();
+
+ /**
+ * Creates the SparkStreaming context.
+ *
+ * @param batchDuration Time interval at which streaming data will be divided into batches,
+ * in milliseconds.
+ */
+ void createStreamingContext(long batchDuration);
+
+ /** Stops the SparkStreaming context. */
+ void stopStreamingCtx();
+
+ /**
+ * Returns a local tmp dir specific to the context
+ */
+ File getLocalTmpDir();
+
+ /**
+ * Returns SparkSession if it existed, otherwise throws Exception.
+ */
+ <E> E sparkSession() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/JobHandle.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/livy/JobHandle.java b/api/src/main/java/org/apache/livy/JobHandle.java
new file mode 100644
index 0000000..d2a71ff
--- /dev/null
+++ b/api/src/main/java/org/apache/livy/JobHandle.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy;
+
+import java.util.List;
+import java.util.concurrent.Future;
+
+/**
+ * A handle to a submitted job. Allows for monitoring and controlling of the running remote job.
+ */
+public interface JobHandle<T> extends Future<T> {
+
+ /**
+ * Return the current state of the job.
+ */
+ State getState();
+
+ /**
+ * Add a listener to the job handle. If the job's state is not SENT, a callback for the
+ * corresponding state will be invoked immediately.
+ *
+ * @param l The listener to add.
+ */
+ void addListener(Listener<T> l);
+
+ /**
+ * The current state of the submitted job.
+ */
+ static enum State {
+ SENT,
+ QUEUED,
+ STARTED,
+ CANCELLED,
+ FAILED,
+ SUCCEEDED;
+ }
+
+ /**
+ * A listener for monitoring the state of the job in the remote context. Callbacks are called
+ * when the corresponding state change occurs.
+ */
+ static interface Listener<T> {
+
+ /**
+ * Notifies when a job has been queued for execution on the remote context. Note that it is
+ * possible for jobs to bypass this state and got directly from the SENT state to the STARTED
+ * state.
+ */
+ void onJobQueued(JobHandle<T> job);
+
+ void onJobStarted(JobHandle<T> job);
+
+ void onJobCancelled(JobHandle<T> job);
+
+ void onJobFailed(JobHandle<T> job, Throwable cause);
+
+ void onJobSucceeded(JobHandle<T> job, T result);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/LivyClient.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/livy/LivyClient.java b/api/src/main/java/org/apache/livy/LivyClient.java
new file mode 100644
index 0000000..72408e4
--- /dev/null
+++ b/api/src/main/java/org/apache/livy/LivyClient.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy;
+
+import java.io.File;
+import java.net.URI;
+import java.util.concurrent.Future;
+
+/**
+ * A client for submitting Spark-based jobs to a Livy backend.
+ */
+public interface LivyClient {
+
+ /**
+ * Submits a job for asynchronous execution.
+ *
+ * @param job The job to execute.
+ * @return A handle that be used to monitor the job.
+ */
+ <T> JobHandle<T> submit(Job<T> job);
+
+ /**
+ * Asks the remote context to run a job immediately.
+ * <p/>
+ * Normally, the remote context will queue jobs and execute them based on how many worker
+ * threads have been configured. This method will run the submitted job in the same thread
+ * processing the RPC message, so that queueing does not apply.
+ * <p/>
+ * It's recommended that this method only be used to run code that finishes quickly. This
+ * avoids interfering with the normal operation of the context.
+ * <p/>
+ * Note: the {@link JobContext#monitor()} functionality is not available when using this method.
+ *
+ * @param job The job to execute.
+ * @return A future to monitor the result of the job.
+ */
+ <T> Future<T> run(Job<T> job);
+
+ /**
+ * Stops the remote context.
+ *
+ * Any pending jobs will be cancelled, and the remote context will be torn down.
+ *
+ * @param shutdownContext Whether to shutdown the underlying Spark context. If false, the
+ * context will keep running and it's still possible to send commands
+ * to it, if the backend being used supports it.
+ */
+ void stop(boolean shutdownContext);
+
+ /**
+ * Upload a jar to be added to the Spark application classpath
+ * @param jar The local file to be uploaded
+ * @return A future that can be used to monitor this operation
+ */
+ Future<?> uploadJar(File jar);
+
+ /**
+ * Adds a jar file to the running remote context.
+ * <p>
+ * Note that the URL should be reachable by the Spark driver process. If running the driver
+ * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
+ * on that node (and not on the client machine).
+ * <p>
+ * If the provided URI has no scheme, it's considered to be relative to the default file system
+ * configured in the Livy server.
+ *
+ * @param uri The location of the jar file.
+ * @return A future that can be used to monitor the operation.
+ */
+ Future<?> addJar(URI uri);
+
+ /**
+ * Upload a file to be passed to the Spark application
+ * @param file The local file to be uploaded
+ * @return A future that can be used to monitor this operation
+ */
+ Future<?> uploadFile(File file);
+
+ /**
+ * Adds a file to the running remote context.
+ * <p>
+ * Note that the URL should be reachable by the Spark driver process. If running the driver
+ * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
+ * on that node (and not on the client machine).
+ * <p>
+ * If the provided URI has no scheme, it's considered to be relative to the default file system
+ * configured in the Livy server.
+ *
+ * @param uri The location of the file.
+ * @return A future that can be used to monitor the operation.
+ */
+ Future<?> addFile(URI uri);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/LivyClientBuilder.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/livy/LivyClientBuilder.java b/api/src/main/java/org/apache/livy/LivyClientBuilder.java
new file mode 100644
index 0000000..1acc1f8
--- /dev/null
+++ b/api/src/main/java/org/apache/livy/LivyClientBuilder.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Map;
+import java.util.Properties;
+import java.util.ServiceLoader;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * A builder for Livy clients.
+ */
+public final class LivyClientBuilder {
+
+ public static final String LIVY_URI_KEY = "livy.uri";
+
+ private final Properties config;
+
+ /**
+ * Creates a new builder that will automatically load the default Livy and Spark configuration
+ * from the classpath.
+ */
+ public LivyClientBuilder() throws IOException {
+ this(true);
+ }
+
+ /**
+ * Creates a new builder that will optionally load the default Livy and Spark configuration
+ * from the classpath.
+ *
+ * Livy client configuration is stored in a file called "livy-client.conf", and Spark client
+ * configuration is stored in a file called "spark-defaults.conf", both in the root of the
+ * application's classpath. Livy configuration takes precedence over Spark's (in case
+ * configuration entries are duplicated), and configuration set in this builder object will
+ * override the values in those files.
+ */
+ public LivyClientBuilder(boolean loadDefaults) throws IOException {
+ this.config = new Properties();
+
+ if (loadDefaults) {
+ String[] confFiles = { "spark-defaults.conf", "livy-client.conf" };
+
+ for (String file : confFiles) {
+ URL url = classLoader().getResource(file);
+ if (url != null) {
+ Reader r = new InputStreamReader(url.openStream(), UTF_8);
+ try {
+ config.load(r);
+ } finally {
+ r.close();
+ }
+ }
+ }
+ }
+ }
+
+ public LivyClientBuilder setURI(URI uri) {
+ config.setProperty(LIVY_URI_KEY, uri.toString());
+ return this;
+ }
+
+ public LivyClientBuilder setConf(String key, String value) {
+ if (value != null) {
+ config.setProperty(key, value);
+ } else {
+ config.remove(key);
+ }
+ return this;
+ }
+
+ public LivyClientBuilder setAll(Map<String, String> props) {
+ config.putAll(props);
+ return this;
+ }
+
+ public LivyClientBuilder setAll(Properties props) {
+ config.putAll(props);
+ return this;
+ }
+
+ public LivyClient build() {
+ String uriStr = config.getProperty(LIVY_URI_KEY);
+ if (uriStr == null) {
+ throw new IllegalArgumentException("URI must be provided.");
+ }
+ URI uri;
+ try {
+ uri = new URI(uriStr);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException("Invalid URI.", e);
+ }
+
+ LivyClient client = null;
+ ServiceLoader<LivyClientFactory> loader = ServiceLoader.load(LivyClientFactory.class,
+ classLoader());
+ if (!loader.iterator().hasNext()) {
+ throw new IllegalStateException("No LivyClientFactory implementation was found.");
+ }
+
+ Exception error = null;
+ for (LivyClientFactory factory : loader) {
+ try {
+ client = factory.createClient(uri, config);
+ } catch (Exception e) {
+ if (!(e instanceof RuntimeException)) {
+ e = new RuntimeException(e);
+ }
+ throw (RuntimeException) e;
+ }
+ if (client != null) {
+ break;
+ }
+ }
+
+ if (client == null) {
+ // Redact any user information from the URI when throwing user-visible exceptions that might
+ // be logged.
+ if (uri.getUserInfo() != null) {
+ try {
+ uri = new URI(uri.getScheme(), "[redacted]", uri.getHost(), uri.getPort(), uri.getPath(),
+ uri.getQuery(), uri.getFragment());
+ } catch (URISyntaxException e) {
+ // Shouldn't really happen.
+ throw new RuntimeException(e);
+ }
+ }
+
+ throw new IllegalArgumentException(String.format(
+ "URI '%s' is not supported by any registered client factories.", uri));
+ }
+ return client;
+ }
+
+ private ClassLoader classLoader() {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ if (cl == null) {
+ cl = getClass().getClassLoader();
+ }
+ return cl;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/LivyClientFactory.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/livy/LivyClientFactory.java b/api/src/main/java/org/apache/livy/LivyClientFactory.java
new file mode 100644
index 0000000..2c51d07
--- /dev/null
+++ b/api/src/main/java/org/apache/livy/LivyClientFactory.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.apache.livy.annotations.Private;
+
+/**
+ * A factory for Livy clients. Client implementations can register themselves by using the
+ * Java services mechanism, providing implementations of this interface.
+ * <p>
+ * Client applications do not need to use this interface directly. Instead, use
+ * {@link LivyClientBuilder}.
+ *
+ * @see java.util.ServiceLoader
+ */
+@Private
+public interface LivyClientFactory {
+
+ /**
+ * Instantiates a new client if the given URI is supported by the implementation.
+ *
+ * @param uri URI pointing at the livy backend to use.
+ */
+ LivyClient createClient(URI uri, Properties config);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/annotations/Private.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/livy/annotations/Private.java b/api/src/main/java/org/apache/livy/annotations/Private.java
new file mode 100644
index 0000000..546103f
--- /dev/null
+++ b/api/src/main/java/org/apache/livy/annotations/Private.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Indicates an API that is considered private to Livy and should not be used by client
+ * applications.
+ */
+@Documented
+@Retention(RetentionPolicy.CLASS)
+public @interface Private { }
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/java/com/cloudera/livy/TestClientFactory.java
----------------------------------------------------------------------
diff --git a/api/src/test/java/com/cloudera/livy/TestClientFactory.java b/api/src/test/java/com/cloudera/livy/TestClientFactory.java
deleted file mode 100644
index c1ff7ac..0000000
--- a/api/src/test/java/com/cloudera/livy/TestClientFactory.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy;
-
-import java.io.File;
-import java.net.URI;
-import java.util.Properties;
-import java.util.concurrent.Future;
-
-public class TestClientFactory implements LivyClientFactory {
-
- @Override
- public LivyClient createClient(URI uri, Properties config) {
- switch (uri.getPath()) {
- case "match":
- return new Client(config);
-
- case "error":
- throw new IllegalStateException("error");
-
- default:
- return null;
- }
- }
-
- public static class Client implements LivyClient {
-
- public final Properties config;
-
- private Client(Properties config) {
- this.config = config;
- }
-
- @Override
- public <T> JobHandle<T> submit(Job<T> job) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public <T> Future<T> run(Job<T> job) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void stop(boolean shutdownContext) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Future<?> uploadJar(File jar) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Future<?> addJar(URI uri) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Future<?> uploadFile(File file) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Future<?> addFile(URI uri) {
- throw new UnsupportedOperationException();
- }
-
-}
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/java/com/cloudera/livy/TestLivyClientBuilder.java
----------------------------------------------------------------------
diff --git a/api/src/test/java/com/cloudera/livy/TestLivyClientBuilder.java b/api/src/test/java/com/cloudera/livy/TestLivyClientBuilder.java
deleted file mode 100644
index 44144ed..0000000
--- a/api/src/test/java/com/cloudera/livy/TestLivyClientBuilder.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy;
-
-import java.net.URI;
-import java.util.Properties;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-public class TestLivyClientBuilder {
-
- @Test
- public void testMatch() throws Exception {
- Properties props = new Properties();
- props.setProperty("prop1", "_prop1_");
- props.setProperty("prop3", "prop3");
-
- TestClientFactory.Client client = (TestClientFactory.Client)
- new LivyClientBuilder(false)
- .setURI(new URI("match"))
- .setConf("prop1", "prop1")
- .setConf("prop2", "prop2")
- .setAll(props)
- .build();
-
- assertNotNull(client);
- assertEquals("_prop1_", client.config.getProperty("prop1"));
- assertEquals("prop2", client.config.getProperty("prop2"));
- assertEquals("prop3", client.config.getProperty("prop3"));
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testMissingUri() throws Exception {
- new LivyClientBuilder(false).build();
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testMismatch() throws Exception {
- assertNull(new LivyClientBuilder(false).setURI(new URI("mismatch")).build());
- }
-
- @Test(expected=IllegalStateException.class)
- public void testFactoryError() throws Exception {
- new LivyClientBuilder(false).setURI(new URI("error")).build();
- }
-
- @Test
- public void testDefaultConfig() throws Exception {
- TestClientFactory.Client client = (TestClientFactory.Client)
- new LivyClientBuilder().build();
- assertEquals("override", client.config.getProperty("spark.config"));
- }
-
- @Test
- public void testRedaction() throws Exception {
- try {
- new LivyClientBuilder(false).setURI(new URI("mismatch://user@host")).build();
- fail("Should have failed to create client.");
- } catch (IllegalArgumentException e) {
- assertFalse("Should have redacted user information.",
- e.getMessage().contains("user@host"));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/java/org/apache/livy/TestClientFactory.java
----------------------------------------------------------------------
diff --git a/api/src/test/java/org/apache/livy/TestClientFactory.java b/api/src/test/java/org/apache/livy/TestClientFactory.java
new file mode 100644
index 0000000..89edeec
--- /dev/null
+++ b/api/src/test/java/org/apache/livy/TestClientFactory.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy;
+
+import java.io.File;
+import java.net.URI;
+import java.util.Properties;
+import java.util.concurrent.Future;
+
+public class TestClientFactory implements LivyClientFactory {
+
+ @Override
+ public LivyClient createClient(URI uri, Properties config) {
+ switch (uri.getPath()) {
+ case "match":
+ return new Client(config);
+
+ case "error":
+ throw new IllegalStateException("error");
+
+ default:
+ return null;
+ }
+ }
+
+ public static class Client implements LivyClient {
+
+ public final Properties config;
+
+ private Client(Properties config) {
+ this.config = config;
+ }
+
+ @Override
+ public <T> JobHandle<T> submit(Job<T> job) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> Future<T> run(Job<T> job) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void stop(boolean shutdownContext) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Future<?> uploadJar(File jar) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Future<?> addJar(URI uri) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Future<?> uploadFile(File file) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Future<?> addFile(URI uri) {
+ throw new UnsupportedOperationException();
+ }
+
+}
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/java/org/apache/livy/TestLivyClientBuilder.java
----------------------------------------------------------------------
diff --git a/api/src/test/java/org/apache/livy/TestLivyClientBuilder.java b/api/src/test/java/org/apache/livy/TestLivyClientBuilder.java
new file mode 100644
index 0000000..568d633
--- /dev/null
+++ b/api/src/test/java/org/apache/livy/TestLivyClientBuilder.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestLivyClientBuilder {
+
+ @Test
+ public void testMatch() throws Exception {
+ Properties props = new Properties();
+ props.setProperty("prop1", "_prop1_");
+ props.setProperty("prop3", "prop3");
+
+ TestClientFactory.Client client = (TestClientFactory.Client)
+ new LivyClientBuilder(false)
+ .setURI(new URI("match"))
+ .setConf("prop1", "prop1")
+ .setConf("prop2", "prop2")
+ .setAll(props)
+ .build();
+
+ assertNotNull(client);
+ assertEquals("_prop1_", client.config.getProperty("prop1"));
+ assertEquals("prop2", client.config.getProperty("prop2"));
+ assertEquals("prop3", client.config.getProperty("prop3"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testMissingUri() throws Exception {
+ new LivyClientBuilder(false).build();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testMismatch() throws Exception {
+ assertNull(new LivyClientBuilder(false).setURI(new URI("mismatch")).build());
+ }
+
+ @Test(expected=IllegalStateException.class)
+ public void testFactoryError() throws Exception {
+ new LivyClientBuilder(false).setURI(new URI("error")).build();
+ }
+
+ @Test
+ public void testDefaultConfig() throws Exception {
+ TestClientFactory.Client client = (TestClientFactory.Client)
+ new LivyClientBuilder().build();
+ assertEquals("override", client.config.getProperty("spark.config"));
+ }
+
+ @Test
+ public void testRedaction() throws Exception {
+ try {
+ new LivyClientBuilder(false).setURI(new URI("mismatch://user@host")).build();
+ fail("Should have failed to create client.");
+ } catch (IllegalArgumentException e) {
+ assertFalse("Should have redacted user information.",
+ e.getMessage().contains("user@host"));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/resources/META-INF/services/com.cloudera.livy.LivyClientFactory
----------------------------------------------------------------------
diff --git a/api/src/test/resources/META-INF/services/com.cloudera.livy.LivyClientFactory b/api/src/test/resources/META-INF/services/com.cloudera.livy.LivyClientFactory
deleted file mode 100644
index 603e146..0000000
--- a/api/src/test/resources/META-INF/services/com.cloudera.livy.LivyClientFactory
+++ /dev/null
@@ -1 +0,0 @@
-com.cloudera.livy.TestClientFactory
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/resources/META-INF/services/org.apache.livy.LivyClientFactory
----------------------------------------------------------------------
diff --git a/api/src/test/resources/META-INF/services/org.apache.livy.LivyClientFactory b/api/src/test/resources/META-INF/services/org.apache.livy.LivyClientFactory
new file mode 100644
index 0000000..99c0bd2
--- /dev/null
+++ b/api/src/test/resources/META-INF/services/org.apache.livy.LivyClientFactory
@@ -0,0 +1 @@
+org.apache.livy.TestClientFactory
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 3b7aa6d..abe9545 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -18,14 +18,14 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>livy-assembly</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/bin/livy-server
----------------------------------------------------------------------
diff --git a/bin/livy-server b/bin/livy-server
index 51d765d..ede63cd 100755
--- a/bin/livy-server
+++ b/bin/livy-server
@@ -92,7 +92,7 @@ start_livy_server() {
LIVY_CLASSPATH="$LIVY_CLASSPATH:$YARN_CONF_DIR"
fi
- command="$RUNNER $LIVY_SERVER_JAVA_OPTS -cp $LIVY_CLASSPATH:$CLASSPATH com.cloudera.livy.server.LivyServer"
+ command="$RUNNER $LIVY_SERVER_JAVA_OPTS -cp $LIVY_CLASSPATH:$CLASSPATH org.apache.livy.server.LivyServer"
if [ $1 = "old" ]; then
exec $command
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/checkstyle.xml
----------------------------------------------------------------------
diff --git a/checkstyle.xml b/checkstyle.xml
index 929f739..76c37a9 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -102,7 +102,7 @@
<module name="ImportOrder">
<property name="separated" value="true"/>
<property name="ordered" value="true"/>
- <property name="groups" value="/^javax?\./,scala,*,com.cloudera.livy"/>
+ <property name="groups" value="/^javax?\./,scala,*,org.apache.livy"/>
</module>
<module name="MethodParamPad"/>
<module name="AnnotationLocation">
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/pom.xml
----------------------------------------------------------------------
diff --git a/client-common/pom.xml b/client-common/pom.xml
index cdf48e6..6f40e0d 100644
--- a/client-common/pom.xml
+++ b/client-common/pom.xml
@@ -18,19 +18,19 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
</parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-client-common</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<dependencies>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-api</artifactId>
<version>${project.version}</version>
</dependency>
[06/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/WebServer.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/WebServer.scala b/server/src/main/scala/org/apache/livy/server/WebServer.scala
new file mode 100644
index 0000000..549d6ab
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/WebServer.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import java.net.InetAddress
+import javax.servlet.ServletContextListener
+
+import org.eclipse.jetty.server._
+import org.eclipse.jetty.server.handler.{HandlerCollection, RequestLogHandler}
+import org.eclipse.jetty.servlet.{DefaultServlet, ServletContextHandler}
+import org.eclipse.jetty.util.ssl.SslContextFactory
+
+import org.apache.livy.{LivyConf, Logging}
+
+class WebServer(livyConf: LivyConf, var host: String, var port: Int) extends Logging {
+ val server = new Server()
+
+ server.setStopTimeout(1000)
+ server.setStopAtShutdown(true)
+
+ val (connector, protocol) = Option(livyConf.get(LivyConf.SSL_KEYSTORE)) match {
+ case None =>
+ val http = new HttpConfiguration()
+ http.setRequestHeaderSize(livyConf.getInt(LivyConf.REQUEST_HEADER_SIZE))
+ http.setResponseHeaderSize(livyConf.getInt(LivyConf.RESPONSE_HEADER_SIZE))
+ (new ServerConnector(server, new HttpConnectionFactory(http)), "http")
+
+ case Some(keystore) =>
+ val https = new HttpConfiguration()
+ https.setRequestHeaderSize(livyConf.getInt(LivyConf.REQUEST_HEADER_SIZE))
+ https.setResponseHeaderSize(livyConf.getInt(LivyConf.RESPONSE_HEADER_SIZE))
+ https.addCustomizer(new SecureRequestCustomizer())
+
+ val sslContextFactory = new SslContextFactory()
+ sslContextFactory.setKeyStorePath(keystore)
+ Option(livyConf.get(LivyConf.SSL_KEYSTORE_PASSWORD))
+ .foreach(sslContextFactory.setKeyStorePassword)
+ Option(livyConf.get(LivyConf.SSL_KEY_PASSWORD))
+ .foreach(sslContextFactory.setKeyManagerPassword)
+
+ (new ServerConnector(server,
+ new SslConnectionFactory(sslContextFactory, "http/1.1"),
+ new HttpConnectionFactory(https)), "https")
+ }
+
+ connector.setHost(host)
+ connector.setPort(port)
+
+ server.setConnectors(Array(connector))
+
+ val context = new ServletContextHandler()
+
+ context.setContextPath("/")
+ context.addServlet(classOf[DefaultServlet], "/")
+
+ val handlers = new HandlerCollection
+ handlers.addHandler(context)
+
+ // Configure the access log
+ val requestLogHandler = new RequestLogHandler
+ val requestLog = new NCSARequestLog(sys.env.getOrElse("LIVY_LOG_DIR",
+ sys.env("LIVY_HOME") + "/logs") + "/yyyy_mm_dd.request.log")
+ requestLog.setAppend(true)
+ requestLog.setExtended(false)
+ requestLog.setLogTimeZone("GMT")
+ requestLog.setRetainDays(livyConf.getInt(LivyConf.REQUEST_LOG_RETAIN_DAYS))
+ requestLogHandler.setRequestLog(requestLog)
+ handlers.addHandler(requestLogHandler)
+
+ server.setHandler(handlers)
+
+ def addEventListener(listener: ServletContextListener): Unit = {
+ context.addEventListener(listener)
+ }
+
+ def start(): Unit = {
+ server.start()
+
+ val connector = server.getConnectors()(0).asInstanceOf[NetworkConnector]
+
+ if (host == "0.0.0.0") {
+ host = InetAddress.getLocalHost.getCanonicalHostName
+ }
+ port = connector.getLocalPort
+
+ info("Starting server on %s://%s:%d" format (protocol, host, port))
+ }
+
+ def join(): Unit = {
+ server.join()
+ }
+
+ def stop(): Unit = {
+ context.stop()
+ server.stop()
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/batch/BatchSession.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/batch/BatchSession.scala b/server/src/main/scala/org/apache/livy/server/batch/BatchSession.scala
new file mode 100644
index 0000000..2605bf5
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/batch/BatchSession.scala
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.batch
+
+import java.lang.ProcessBuilder.Redirect
+
+import scala.concurrent.{ExecutionContext, ExecutionContextExecutor}
+import scala.util.Random
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.{Session, SessionState}
+import org.apache.livy.sessions.Session._
+import org.apache.livy.utils.{AppInfo, SparkApp, SparkAppListener, SparkProcessBuilder}
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+case class BatchRecoveryMetadata(
+ id: Int,
+ appId: Option[String],
+ appTag: String,
+ owner: String,
+ proxyUser: Option[String],
+ version: Int = 1)
+ extends RecoveryMetadata
+
+object BatchSession extends Logging {
+ val RECOVERY_SESSION_TYPE = "batch"
+
+ def create(
+ id: Int,
+ request: CreateBatchRequest,
+ livyConf: LivyConf,
+ owner: String,
+ proxyUser: Option[String],
+ sessionStore: SessionStore,
+ mockApp: Option[SparkApp] = None): BatchSession = {
+ val appTag = s"livy-batch-$id-${Random.alphanumeric.take(8).mkString}"
+
+ def createSparkApp(s: BatchSession): SparkApp = {
+ val conf = SparkApp.prepareSparkConf(
+ appTag,
+ livyConf,
+ prepareConf(
+ request.conf, request.jars, request.files, request.archives, request.pyFiles, livyConf))
+ require(request.file != null, "File is required.")
+
+ val builder = new SparkProcessBuilder(livyConf)
+ builder.conf(conf)
+
+ proxyUser.foreach(builder.proxyUser)
+ request.className.foreach(builder.className)
+ request.driverMemory.foreach(builder.driverMemory)
+ request.driverCores.foreach(builder.driverCores)
+ request.executorMemory.foreach(builder.executorMemory)
+ request.executorCores.foreach(builder.executorCores)
+ request.numExecutors.foreach(builder.numExecutors)
+ request.queue.foreach(builder.queue)
+ request.name.foreach(builder.name)
+
+ // Spark 1.x does not support specifying deploy mode in conf and needs special handling.
+ livyConf.sparkDeployMode().foreach(builder.deployMode)
+
+ sessionStore.save(BatchSession.RECOVERY_SESSION_TYPE, s.recoveryMetadata)
+
+ builder.redirectOutput(Redirect.PIPE)
+ builder.redirectErrorStream(true)
+
+ val file = resolveURIs(Seq(request.file), livyConf)(0)
+ val sparkSubmit = builder.start(Some(file), request.args)
+
+ SparkApp.create(appTag, None, Option(sparkSubmit), livyConf, Option(s))
+ }
+
+ info(s"Creating batch session $id: [owner: $owner, request: $request]")
+
+ new BatchSession(
+ id,
+ appTag,
+ SessionState.Starting(),
+ livyConf,
+ owner,
+ proxyUser,
+ sessionStore,
+ mockApp.map { m => (_: BatchSession) => m }.getOrElse(createSparkApp))
+ }
+
+ def recover(
+ m: BatchRecoveryMetadata,
+ livyConf: LivyConf,
+ sessionStore: SessionStore,
+ mockApp: Option[SparkApp] = None): BatchSession = {
+ new BatchSession(
+ m.id,
+ m.appTag,
+ SessionState.Recovering(),
+ livyConf,
+ m.owner,
+ m.proxyUser,
+ sessionStore,
+ mockApp.map { m => (_: BatchSession) => m }.getOrElse { s =>
+ SparkApp.create(m.appTag, m.appId, None, livyConf, Option(s))
+ })
+ }
+}
+
+class BatchSession(
+ id: Int,
+ appTag: String,
+ initialState: SessionState,
+ livyConf: LivyConf,
+ owner: String,
+ override val proxyUser: Option[String],
+ sessionStore: SessionStore,
+ sparkApp: BatchSession => SparkApp)
+ extends Session(id, owner, livyConf) with SparkAppListener {
+ import BatchSession._
+
+ protected implicit def executor: ExecutionContextExecutor = ExecutionContext.global
+
+ private[this] var _state: SessionState = initialState
+ private val app = sparkApp(this)
+
+ override def state: SessionState = _state
+
+ override def logLines(): IndexedSeq[String] = app.log()
+
+ override def stopSession(): Unit = {
+ app.kill()
+ }
+
+ override def appIdKnown(appId: String): Unit = {
+ _appId = Option(appId)
+ sessionStore.save(RECOVERY_SESSION_TYPE, recoveryMetadata)
+ }
+
+ override def stateChanged(oldState: SparkApp.State, newState: SparkApp.State): Unit = {
+ synchronized {
+ debug(s"$this state changed from $oldState to $newState")
+ newState match {
+ case SparkApp.State.RUNNING =>
+ _state = SessionState.Running()
+ info(s"Batch session $id created [appid: ${appId.orNull}, state: ${state.toString}, " +
+ s"info: ${appInfo.asJavaMap}]")
+ case SparkApp.State.FINISHED => _state = SessionState.Success()
+ case SparkApp.State.KILLED | SparkApp.State.FAILED =>
+ _state = SessionState.Dead()
+ case _ =>
+ }
+ }
+ }
+
+ override def infoChanged(appInfo: AppInfo): Unit = { this.appInfo = appInfo }
+
+ override def recoveryMetadata: RecoveryMetadata =
+ BatchRecoveryMetadata(id, appId, appTag, owner, proxyUser)
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/batch/BatchSessionServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/batch/BatchSessionServlet.scala b/server/src/main/scala/org/apache/livy/server/batch/BatchSessionServlet.scala
new file mode 100644
index 0000000..db98a24
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/batch/BatchSessionServlet.scala
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.batch
+
+import javax.servlet.http.HttpServletRequest
+
+import org.apache.livy.LivyConf
+import org.apache.livy.server.SessionServlet
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.BatchSessionManager
+import org.apache.livy.utils.AppInfo
+
+case class BatchSessionView(
+ id: Long,
+ state: String,
+ appId: Option[String],
+ appInfo: AppInfo,
+ log: Seq[String])
+
+class BatchSessionServlet(
+ sessionManager: BatchSessionManager,
+ sessionStore: SessionStore,
+ livyConf: LivyConf)
+ extends SessionServlet(sessionManager, livyConf)
+{
+
+ override protected def createSession(req: HttpServletRequest): BatchSession = {
+ val createRequest = bodyAs[CreateBatchRequest](req)
+ val proxyUser = checkImpersonation(createRequest.proxyUser, req)
+ BatchSession.create(
+ sessionManager.nextId(), createRequest, livyConf, remoteUser(req), proxyUser, sessionStore)
+ }
+
+ override protected[batch] def clientSessionView(
+ session: BatchSession,
+ req: HttpServletRequest): Any = {
+ val logs =
+ if (hasAccess(session.owner, req)) {
+ val lines = session.logLines()
+
+ val size = 10
+ val from = math.max(0, lines.length - size)
+ val until = from + size
+
+ lines.view(from, until).toSeq
+ } else {
+ Nil
+ }
+ BatchSessionView(session.id, session.state.toString, session.appId, session.appInfo, logs)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala b/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala
new file mode 100644
index 0000000..53b5e1b
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.batch
+
+class CreateBatchRequest {
+
+ var file: String = _
+ var proxyUser: Option[String] = None
+ var args: List[String] = List()
+ var className: Option[String] = None
+ var jars: List[String] = List()
+ var pyFiles: List[String] = List()
+ var files: List[String] = List()
+ var driverMemory: Option[String] = None
+ var driverCores: Option[Int] = None
+ var executorMemory: Option[String] = None
+ var executorCores: Option[Int] = None
+ var numExecutors: Option[Int] = None
+ var archives: List[String] = List()
+ var queue: Option[String] = None
+ var name: Option[String] = None
+ var conf: Map[String, String] = Map()
+
+ override def toString: String = {
+ s"[proxyUser: $proxyUser, " +
+ s"file: $file, " +
+ (if (args.nonEmpty) s"args: ${args.mkString(",")}, " else "") +
+ (if (jars.nonEmpty) s"jars: ${jars.mkString(",")}, " else "") +
+ (if (pyFiles.nonEmpty) s"pyFiles: ${pyFiles.mkString(",")}, " else "") +
+ (if (files.nonEmpty) s"files: ${files.mkString(",")}, " else "") +
+ (if (archives.nonEmpty) s"archives: ${archives.mkString(",")}, " else "") +
+ (if (driverMemory.isDefined) s"driverMemory: ${driverMemory.get}, " else "") +
+ (if (driverCores.isDefined) s"driverCores: ${driverCores.get}, " else "") +
+ (if (executorMemory.isDefined) s"executorMemory: ${executorMemory.get}, " else "") +
+ (if (executorCores.isDefined) s"executorCores: ${executorCores.get}, " else "") +
+ (if (numExecutors.isDefined) s"numExecutors: ${numExecutors.get}, " else "") +
+ (if (queue.isDefined) s"queue: ${queue.get}, " else "") +
+ (if (name.isDefined) s"name: ${name.get}, " else "") +
+ (if (conf.nonEmpty) s"conf: ${conf.mkString(",")}]" else "]")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala b/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala
new file mode 100644
index 0000000..bbb7abd
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.interactive
+
+import org.apache.livy.sessions.{Kind, Spark}
+
+class CreateInteractiveRequest {
+ var kind: Kind = Spark()
+ var proxyUser: Option[String] = None
+ var jars: List[String] = List()
+ var pyFiles: List[String] = List()
+ var files: List[String] = List()
+ var driverMemory: Option[String] = None
+ var driverCores: Option[Int] = None
+ var executorMemory: Option[String] = None
+ var executorCores: Option[Int] = None
+ var numExecutors: Option[Int] = None
+ var archives: List[String] = List()
+ var queue: Option[String] = None
+ var name: Option[String] = None
+ var conf: Map[String, String] = Map()
+ var heartbeatTimeoutInSecond: Int = 0
+
+ override def toString: String = {
+ s"[kind: $kind, " +
+ s"proxyUser: $proxyUser, " +
+ (if (jars.nonEmpty) s"jars: ${jars.mkString(",")}, " else "") +
+ (if (pyFiles.nonEmpty) s"pyFiles: ${pyFiles.mkString(",")}, " else "") +
+ (if (files.nonEmpty) s"files: ${files.mkString(",")}, " else "") +
+ (if (archives.nonEmpty) s"archives: ${archives.mkString(",")}, " else "") +
+ (if (driverMemory.isDefined) s"driverMemory: ${driverMemory.get}, " else "") +
+ (if (driverCores.isDefined) s"driverCores: ${driverCores.get}, " else "") +
+ (if (executorMemory.isDefined) s"executorMemory: ${executorMemory.get}, " else "") +
+ (if (executorCores.isDefined) s"executorCores: ${executorCores.get}, " else "") +
+ (if (numExecutors.isDefined) s"numExecutors: ${numExecutors.get}, " else "") +
+ (if (queue.isDefined) s"queue: ${queue.get}, " else "") +
+ (if (name.isDefined) s"name: ${name.get}, " else "") +
+ (if (conf.nonEmpty) s"conf: ${conf.mkString(",")}, " else "") +
+ s"heartbeatTimeoutInSecond: $heartbeatTimeoutInSecond]"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
new file mode 100644
index 0000000..b466d85
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
@@ -0,0 +1,608 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.interactive
+
+import java.io.{File, InputStream}
+import java.net.URI
+import java.nio.ByteBuffer
+import java.nio.file.{Files, Paths}
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicLong
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.concurrent.Future
+import scala.concurrent.duration.{Duration, FiniteDuration}
+import scala.util.Random
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.google.common.annotations.VisibleForTesting
+import org.apache.hadoop.fs.Path
+import org.apache.spark.launcher.SparkLauncher
+
+import org.apache.livy._
+import org.apache.livy.client.common.HttpMessages._
+import org.apache.livy.rsc.{PingJob, RSCClient, RSCConf}
+import org.apache.livy.rsc.driver.Statement
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions._
+import org.apache.livy.sessions.Session._
+import org.apache.livy.sessions.SessionState.Dead
+import org.apache.livy.utils._
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+case class InteractiveRecoveryMetadata(
+ id: Int,
+ appId: Option[String],
+ appTag: String,
+ kind: Kind,
+ heartbeatTimeoutS: Int,
+ owner: String,
+ proxyUser: Option[String],
+ rscDriverUri: Option[URI],
+ version: Int = 1)
+ extends RecoveryMetadata
+
+object InteractiveSession extends Logging {
+ private[interactive] val SPARK_YARN_IS_PYTHON = "spark.yarn.isPython"
+
+ val RECOVERY_SESSION_TYPE = "interactive"
+
+ def create(
+ id: Int,
+ owner: String,
+ proxyUser: Option[String],
+ livyConf: LivyConf,
+ request: CreateInteractiveRequest,
+ sessionStore: SessionStore,
+ mockApp: Option[SparkApp] = None,
+ mockClient: Option[RSCClient] = None): InteractiveSession = {
+ val appTag = s"livy-session-$id-${Random.alphanumeric.take(8).mkString}"
+
+ val client = mockClient.orElse {
+ val conf = SparkApp.prepareSparkConf(appTag, livyConf, prepareConf(
+ request.conf, request.jars, request.files, request.archives, request.pyFiles, livyConf))
+
+ val builderProperties = prepareBuilderProp(conf, request.kind, livyConf)
+
+ val userOpts: Map[String, Option[String]] = Map(
+ "spark.driver.cores" -> request.driverCores.map(_.toString),
+ SparkLauncher.DRIVER_MEMORY -> request.driverMemory.map(_.toString),
+ SparkLauncher.EXECUTOR_CORES -> request.executorCores.map(_.toString),
+ SparkLauncher.EXECUTOR_MEMORY -> request.executorMemory.map(_.toString),
+ "spark.executor.instances" -> request.numExecutors.map(_.toString),
+ "spark.app.name" -> request.name.map(_.toString),
+ "spark.yarn.queue" -> request.queue
+ )
+
+ userOpts.foreach { case (key, opt) =>
+ opt.foreach { value => builderProperties.put(key, value) }
+ }
+
+ builderProperties.getOrElseUpdate("spark.app.name", s"livy-session-$id")
+
+ info(s"Creating Interactive session $id: [owner: $owner, request: $request]")
+ val builder = new LivyClientBuilder()
+ .setAll(builderProperties.asJava)
+ .setConf("livy.client.session-id", id.toString)
+ .setConf(RSCConf.Entry.DRIVER_CLASS.key(), "org.apache.livy.repl.ReplDriver")
+ .setConf(RSCConf.Entry.PROXY_USER.key(), proxyUser.orNull)
+ .setURI(new URI("rsc:/"))
+
+ Option(builder.build().asInstanceOf[RSCClient])
+ }
+
+ new InteractiveSession(
+ id,
+ None,
+ appTag,
+ client,
+ SessionState.Starting(),
+ request.kind,
+ request.heartbeatTimeoutInSecond,
+ livyConf,
+ owner,
+ proxyUser,
+ sessionStore,
+ mockApp)
+ }
+
+ def recover(
+ metadata: InteractiveRecoveryMetadata,
+ livyConf: LivyConf,
+ sessionStore: SessionStore,
+ mockApp: Option[SparkApp] = None,
+ mockClient: Option[RSCClient] = None): InteractiveSession = {
+ val client = mockClient.orElse(metadata.rscDriverUri.map { uri =>
+ val builder = new LivyClientBuilder().setURI(uri)
+ builder.build().asInstanceOf[RSCClient]
+ })
+
+ new InteractiveSession(
+ metadata.id,
+ metadata.appId,
+ metadata.appTag,
+ client,
+ SessionState.Recovering(),
+ metadata.kind,
+ metadata.heartbeatTimeoutS,
+ livyConf,
+ metadata.owner,
+ metadata.proxyUser,
+ sessionStore,
+ mockApp)
+ }
+
+ @VisibleForTesting
+ private[interactive] def prepareBuilderProp(
+ conf: Map[String, String],
+ kind: Kind,
+ livyConf: LivyConf): mutable.Map[String, String] = {
+
+ val builderProperties = mutable.Map[String, String]()
+ builderProperties ++= conf
+
+ def livyJars(livyConf: LivyConf, scalaVersion: String): List[String] = {
+ Option(livyConf.get(LivyConf.REPL_JARS)).map { jars =>
+ val regex = """[\w-]+_(\d\.\d\d).*\.jar""".r
+ jars.split(",").filter { name => new Path(name).getName match {
+ // Filter out unmatched scala jars
+ case regex(ver) => ver == scalaVersion
+ // Keep all the java jars end with ".jar"
+ case _ => name.endsWith(".jar")
+ }
+ }.toList
+ }.getOrElse {
+ val home = sys.env("LIVY_HOME")
+ val jars = Option(new File(home, s"repl_$scalaVersion-jars"))
+ .filter(_.isDirectory())
+ .getOrElse(new File(home, s"repl/scala-$scalaVersion/target/jars"))
+ require(jars.isDirectory(), "Cannot find Livy REPL jars.")
+ jars.listFiles().map(_.getAbsolutePath()).toList
+ }
+ }
+
+ def findSparkRArchive(): Option[String] = {
+ Option(livyConf.get(RSCConf.Entry.SPARKR_PACKAGE.key())).orElse {
+ sys.env.get("SPARK_HOME").map { case sparkHome =>
+ val path = Seq(sparkHome, "R", "lib", "sparkr.zip").mkString(File.separator)
+ val rArchivesFile = new File(path)
+ require(rArchivesFile.exists(), "sparkr.zip not found; cannot run sparkr application.")
+ rArchivesFile.getAbsolutePath()
+ }
+ }
+ }
+
+ def datanucleusJars(livyConf: LivyConf, sparkMajorVersion: Int): Seq[String] = {
+ if (sys.env.getOrElse("LIVY_INTEGRATION_TEST", "false").toBoolean) {
+ // datanucleus jars has already been in classpath in integration test
+ Seq.empty
+ } else {
+ val sparkHome = livyConf.sparkHome().get
+ val libdir = sparkMajorVersion match {
+ case 1 =>
+ if (new File(sparkHome, "RELEASE").isFile) {
+ new File(sparkHome, "lib")
+ } else {
+ new File(sparkHome, "lib_managed/jars")
+ }
+ case 2 =>
+ if (new File(sparkHome, "RELEASE").isFile) {
+ new File(sparkHome, "jars")
+ } else if (new File(sparkHome, "assembly/target/scala-2.11/jars").isDirectory) {
+ new File(sparkHome, "assembly/target/scala-2.11/jars")
+ } else {
+ new File(sparkHome, "assembly/target/scala-2.10/jars")
+ }
+ case v =>
+ throw new RuntimeException("Unsupported spark major version:" + sparkMajorVersion)
+ }
+ val jars = if (!libdir.isDirectory) {
+ Seq.empty[String]
+ } else {
+ libdir.listFiles().filter(_.getName.startsWith("datanucleus-"))
+ .map(_.getAbsolutePath).toSeq
+ }
+ if (jars.isEmpty) {
+ warn("datanucleus jars can not be found")
+ }
+ jars
+ }
+ }
+
+ /**
+ * Look for hive-site.xml (for now just ignore spark.files defined in spark-defaults.conf)
+ * 1. First look for hive-site.xml in user request
+ * 2. Then look for that under classpath
+ * @param livyConf
+ * @return (hive-site.xml path, whether it is provided by user)
+ */
+ def hiveSiteFile(sparkFiles: Array[String], livyConf: LivyConf): (Option[File], Boolean) = {
+ if (sparkFiles.exists(_.split("/").last == "hive-site.xml")) {
+ (None, true)
+ } else {
+ val hiveSiteURL = getClass.getResource("/hive-site.xml")
+ if (hiveSiteURL != null && hiveSiteURL.getProtocol == "file") {
+ (Some(new File(hiveSiteURL.toURI)), false)
+ } else {
+ (None, false)
+ }
+ }
+ }
+
+ def findPySparkArchives(): Seq[String] = {
+ Option(livyConf.get(RSCConf.Entry.PYSPARK_ARCHIVES))
+ .map(_.split(",").toSeq)
+ .getOrElse {
+ sys.env.get("SPARK_HOME") .map { case sparkHome =>
+ val pyLibPath = Seq(sparkHome, "python", "lib").mkString(File.separator)
+ val pyArchivesFile = new File(pyLibPath, "pyspark.zip")
+ require(pyArchivesFile.exists(),
+ "pyspark.zip not found; cannot run pyspark application in YARN mode.")
+
+ val py4jFile = Files.newDirectoryStream(Paths.get(pyLibPath), "py4j-*-src.zip")
+ .iterator()
+ .next()
+ .toFile
+
+ require(py4jFile.exists(),
+ "py4j-*-src.zip not found; cannot run pyspark application in YARN mode.")
+ Seq(pyArchivesFile.getAbsolutePath, py4jFile.getAbsolutePath)
+ }.getOrElse(Seq())
+ }
+ }
+
+ def mergeConfList(list: Seq[String], key: String): Unit = {
+ if (list.nonEmpty) {
+ builderProperties.get(key) match {
+ case None =>
+ builderProperties.put(key, list.mkString(","))
+ case Some(oldList) =>
+ val newList = (oldList :: list.toList).mkString(",")
+ builderProperties.put(key, newList)
+ }
+ }
+ }
+
+ def mergeHiveSiteAndHiveDeps(sparkMajorVersion: Int): Unit = {
+ val sparkFiles = conf.get("spark.files").map(_.split(",")).getOrElse(Array.empty[String])
+ hiveSiteFile(sparkFiles, livyConf) match {
+ case (_, true) =>
+ debug("Enable HiveContext because hive-site.xml is found in user request.")
+ mergeConfList(datanucleusJars(livyConf, sparkMajorVersion), LivyConf.SPARK_JARS)
+ case (Some(file), false) =>
+ debug("Enable HiveContext because hive-site.xml is found under classpath, "
+ + file.getAbsolutePath)
+ mergeConfList(List(file.getAbsolutePath), LivyConf.SPARK_FILES)
+ mergeConfList(datanucleusJars(livyConf, sparkMajorVersion), LivyConf.SPARK_JARS)
+ case (None, false) =>
+ warn("Enable HiveContext but no hive-site.xml found under" +
+ " classpath or user request.")
+ }
+ }
+
+ kind match {
+ case PySpark() | PySpark3() =>
+ val pySparkFiles = if (!LivyConf.TEST_MODE) findPySparkArchives() else Nil
+ mergeConfList(pySparkFiles, LivyConf.SPARK_PY_FILES)
+ builderProperties.put(SPARK_YARN_IS_PYTHON, "true")
+ case SparkR() =>
+ val sparkRArchive = if (!LivyConf.TEST_MODE) findSparkRArchive() else None
+ sparkRArchive.foreach { archive =>
+ builderProperties.put(RSCConf.Entry.SPARKR_PACKAGE.key(), archive + "#sparkr")
+ }
+ case _ =>
+ }
+ builderProperties.put(RSCConf.Entry.SESSION_KIND.key, kind.toString)
+
+ // Set Livy.rsc.jars from livy conf to rsc conf, RSC conf will take precedence if both are set.
+ Option(livyConf.get(LivyConf.RSC_JARS)).foreach(
+ builderProperties.getOrElseUpdate(RSCConf.Entry.LIVY_JARS.key(), _))
+
+ require(livyConf.get(LivyConf.LIVY_SPARK_VERSION) != null)
+ require(livyConf.get(LivyConf.LIVY_SPARK_SCALA_VERSION) != null)
+
+ val (sparkMajorVersion, _) =
+ LivySparkUtils.formatSparkVersion(livyConf.get(LivyConf.LIVY_SPARK_VERSION))
+ val scalaVersion = livyConf.get(LivyConf.LIVY_SPARK_SCALA_VERSION)
+
+ mergeConfList(livyJars(livyConf, scalaVersion), LivyConf.SPARK_JARS)
+ val enableHiveContext = livyConf.getBoolean(LivyConf.ENABLE_HIVE_CONTEXT)
+ // pass spark.livy.spark_major_version to driver
+ builderProperties.put("spark.livy.spark_major_version", sparkMajorVersion.toString)
+
+ if (sparkMajorVersion <= 1) {
+ builderProperties.put("spark.repl.enableHiveContext",
+ livyConf.getBoolean(LivyConf.ENABLE_HIVE_CONTEXT).toString)
+ } else {
+ val confVal = if (enableHiveContext) "hive" else "in-memory"
+ builderProperties.put("spark.sql.catalogImplementation", confVal)
+ }
+
+ if (enableHiveContext) {
+ mergeHiveSiteAndHiveDeps(sparkMajorVersion)
+ }
+
+ builderProperties
+ }
+}
+
+class InteractiveSession(
+ id: Int,
+ appIdHint: Option[String],
+ appTag: String,
+ client: Option[RSCClient],
+ initialState: SessionState,
+ val kind: Kind,
+ heartbeatTimeoutS: Int,
+ livyConf: LivyConf,
+ owner: String,
+ override val proxyUser: Option[String],
+ sessionStore: SessionStore,
+ mockApp: Option[SparkApp]) // For unit test.
+ extends Session(id, owner, livyConf)
+ with SessionHeartbeat
+ with SparkAppListener {
+
+ import InteractiveSession._
+
+ private var serverSideState: SessionState = initialState
+
+ override protected val heartbeatTimeout: FiniteDuration = {
+ val heartbeatTimeoutInSecond = heartbeatTimeoutS
+ Duration(heartbeatTimeoutInSecond, TimeUnit.SECONDS)
+ }
+ private val operations = mutable.Map[Long, String]()
+ private val operationCounter = new AtomicLong(0)
+ private var rscDriverUri: Option[URI] = None
+ private var sessionLog: IndexedSeq[String] = IndexedSeq.empty
+ private val sessionSaveLock = new Object()
+
+ _appId = appIdHint
+ sessionStore.save(RECOVERY_SESSION_TYPE, recoveryMetadata)
+ heartbeat()
+
+ private val app = mockApp.orElse {
+ if (livyConf.isRunningOnYarn()) {
+ val driverProcess = client.flatMap { c => Option(c.getDriverProcess) }
+ .map(new LineBufferedProcess(_))
+ // When Livy is running with YARN, SparkYarnApp can provide better YARN integration.
+ // (e.g. Reflect YARN application state to session state).
+ Option(SparkApp.create(appTag, appId, driverProcess, livyConf, Some(this)))
+ } else {
+ // When Livy is running with other cluster manager, SparkApp doesn't provide any
+ // additional benefit over controlling RSCDriver using RSCClient. Don't use it.
+ None
+ }
+ }
+
+ if (client.isEmpty) {
+ transition(Dead())
+ val msg = s"Cannot recover interactive session $id because its RSCDriver URI is unknown."
+ info(msg)
+ sessionLog = IndexedSeq(msg)
+ } else {
+ val uriFuture = Future { client.get.getServerUri.get() }
+
+ uriFuture onSuccess { case url =>
+ rscDriverUri = Option(url)
+ sessionSaveLock.synchronized {
+ sessionStore.save(RECOVERY_SESSION_TYPE, recoveryMetadata)
+ }
+ }
+ uriFuture onFailure { case e => warn("Fail to get rsc uri", e) }
+
+ // Send a dummy job that will return once the client is ready to be used, and set the
+ // state to "idle" at that point.
+ client.get.submit(new PingJob()).addListener(new JobHandle.Listener[Void]() {
+ override def onJobQueued(job: JobHandle[Void]): Unit = { }
+ override def onJobStarted(job: JobHandle[Void]): Unit = { }
+
+ override def onJobCancelled(job: JobHandle[Void]): Unit = errorOut()
+
+ override def onJobFailed(job: JobHandle[Void], cause: Throwable): Unit = errorOut()
+
+ override def onJobSucceeded(job: JobHandle[Void], result: Void): Unit = {
+ transition(SessionState.Running())
+ info(s"Interactive session $id created [appid: ${appId.orNull}, owner: $owner, proxyUser:" +
+ s" $proxyUser, state: ${state.toString}, kind: ${kind.toString}, " +
+ s"info: ${appInfo.asJavaMap}]")
+ }
+
+ private def errorOut(): Unit = {
+ // Other code might call stop() to close the RPC channel. When RPC channel is closing,
+ // this callback might be triggered. Check and don't call stop() to avoid nested called
+ // if the session is already shutting down.
+ if (serverSideState != SessionState.ShuttingDown()) {
+ transition(SessionState.Error())
+ stop()
+ app.foreach { a =>
+ info(s"Failed to ping RSC driver for session $id. Killing application.")
+ a.kill()
+ }
+ }
+ }
+ })
+ }
+
+ override def logLines(): IndexedSeq[String] = app.map(_.log()).getOrElse(sessionLog)
+
+ override def recoveryMetadata: RecoveryMetadata =
+ InteractiveRecoveryMetadata(
+ id, appId, appTag, kind, heartbeatTimeout.toSeconds.toInt, owner, proxyUser, rscDriverUri)
+
+ override def state: SessionState = {
+ if (serverSideState.isInstanceOf[SessionState.Running]) {
+ // If session is in running state, return the repl state from RSCClient.
+ client
+ .flatMap(s => Option(s.getReplState))
+ .map(SessionState(_))
+ .getOrElse(SessionState.Busy()) // If repl state is unknown, assume repl is busy.
+ } else {
+ serverSideState
+ }
+ }
+
+ override def stopSession(): Unit = {
+ try {
+ transition(SessionState.ShuttingDown())
+ sessionStore.remove(RECOVERY_SESSION_TYPE, id)
+ client.foreach { _.stop(true) }
+ } catch {
+ case _: Exception =>
+ app.foreach {
+ warn(s"Failed to stop RSCDriver. Killing it...")
+ _.kill()
+ }
+ } finally {
+ transition(SessionState.Dead())
+ }
+ }
+
+ def statements: IndexedSeq[Statement] = {
+ ensureActive()
+ val r = client.get.getReplJobResults().get()
+ r.statements.toIndexedSeq
+ }
+
+ def getStatement(stmtId: Int): Option[Statement] = {
+ ensureActive()
+ val r = client.get.getReplJobResults(stmtId, 1).get()
+ if (r.statements.length < 1) {
+ None
+ } else {
+ Option(r.statements(0))
+ }
+ }
+
+ def interrupt(): Future[Unit] = {
+ stop()
+ }
+
+ def executeStatement(content: ExecuteRequest): Statement = {
+ ensureRunning()
+ recordActivity()
+
+ val id = client.get.submitReplCode(content.code).get
+ client.get.getReplJobResults(id, 1).get().statements(0)
+ }
+
+ def cancelStatement(statementId: Int): Unit = {
+ ensureRunning()
+ recordActivity()
+ client.get.cancelReplCode(statementId)
+ }
+
+ def runJob(job: Array[Byte]): Long = {
+ performOperation(job, true)
+ }
+
+ def submitJob(job: Array[Byte]): Long = {
+ performOperation(job, false)
+ }
+
+ def addFile(fileStream: InputStream, fileName: String): Unit = {
+ addFile(copyResourceToHDFS(fileStream, fileName))
+ }
+
+ def addJar(jarStream: InputStream, jarName: String): Unit = {
+ addJar(copyResourceToHDFS(jarStream, jarName))
+ }
+
+ def addFile(uri: URI): Unit = {
+ ensureActive()
+ recordActivity()
+ client.get.addFile(resolveURI(uri, livyConf)).get()
+ }
+
+ def addJar(uri: URI): Unit = {
+ ensureActive()
+ recordActivity()
+ client.get.addJar(resolveURI(uri, livyConf)).get()
+ }
+
+ def jobStatus(id: Long): Any = {
+ ensureActive()
+ val clientJobId = operations(id)
+ recordActivity()
+ // TODO: don't block indefinitely?
+ val status = client.get.getBypassJobStatus(clientJobId).get()
+ new JobStatus(id, status.state, status.result, status.error)
+ }
+
+ def cancelJob(id: Long): Unit = {
+ ensureActive()
+ recordActivity()
+ operations.remove(id).foreach { client.get.cancel }
+ }
+
+ private def transition(newState: SessionState) = synchronized {
+ // When a statement returns an error, the session should transit to error state.
+ // If the session crashed because of the error, the session should instead go to dead state.
+ // Since these 2 transitions are triggered by different threads, there's a race condition.
+ // Make sure we won't transit from dead to error state.
+ val areSameStates = serverSideState.getClass() == newState.getClass()
+ val transitFromInactiveToActive = !serverSideState.isActive && newState.isActive
+ if (!areSameStates && !transitFromInactiveToActive) {
+ debug(s"$this session state change from ${serverSideState} to $newState")
+ serverSideState = newState
+ }
+ }
+
+ private def ensureActive(): Unit = synchronized {
+ require(serverSideState.isActive, "Session isn't active.")
+ require(client.isDefined, "Session is active but client hasn't been created.")
+ }
+
+ private def ensureRunning(): Unit = synchronized {
+ serverSideState match {
+ case SessionState.Running() =>
+ case _ =>
+ throw new IllegalStateException("Session is in state %s" format serverSideState)
+ }
+ }
+
+ private def performOperation(job: Array[Byte], sync: Boolean): Long = {
+ ensureActive()
+ recordActivity()
+ val future = client.get.bypass(ByteBuffer.wrap(job), sync)
+ val opId = operationCounter.incrementAndGet()
+ operations(opId) = future
+ opId
+ }
+
+ override def appIdKnown(appId: String): Unit = {
+ _appId = Option(appId)
+ sessionSaveLock.synchronized {
+ sessionStore.save(RECOVERY_SESSION_TYPE, recoveryMetadata)
+ }
+ }
+
+ override def stateChanged(oldState: SparkApp.State, newState: SparkApp.State): Unit = {
+ synchronized {
+ debug(s"$this app state changed from $oldState to $newState")
+ newState match {
+ case SparkApp.State.FINISHED | SparkApp.State.KILLED | SparkApp.State.FAILED =>
+ transition(SessionState.Dead())
+ case _ =>
+ }
+ }
+ }
+
+ override def infoChanged(appInfo: AppInfo): Unit = { this.appInfo = appInfo }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSessionServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSessionServlet.scala b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSessionServlet.scala
new file mode 100644
index 0000000..3066a98
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSessionServlet.scala
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.interactive
+
+import java.net.URI
+import javax.servlet.http.HttpServletRequest
+
+import scala.collection.JavaConverters._
+import scala.concurrent._
+import scala.concurrent.duration._
+
+import org.json4s.jackson.Json4sScalaModule
+import org.scalatra._
+import org.scalatra.servlet.FileUploadSupport
+
+import org.apache.livy.{ExecuteRequest, JobHandle, LivyConf, Logging}
+import org.apache.livy.client.common.HttpMessages
+import org.apache.livy.client.common.HttpMessages._
+import org.apache.livy.rsc.driver.Statement
+import org.apache.livy.server.SessionServlet
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions._
+
+object InteractiveSessionServlet extends Logging
+
+class InteractiveSessionServlet(
+ sessionManager: InteractiveSessionManager,
+ sessionStore: SessionStore,
+ livyConf: LivyConf)
+ extends SessionServlet(sessionManager, livyConf)
+ with SessionHeartbeatNotifier[InteractiveSession, InteractiveRecoveryMetadata]
+ with FileUploadSupport
+{
+
+ mapper.registerModule(new SessionKindModule())
+ .registerModule(new Json4sScalaModule())
+
+ override protected def createSession(req: HttpServletRequest): InteractiveSession = {
+ val createRequest = bodyAs[CreateInteractiveRequest](req)
+ val proxyUser = checkImpersonation(createRequest.proxyUser, req)
+ InteractiveSession.create(
+ sessionManager.nextId(),
+ remoteUser(req),
+ proxyUser,
+ livyConf,
+ createRequest,
+ sessionStore)
+ }
+
+ override protected[interactive] def clientSessionView(
+ session: InteractiveSession,
+ req: HttpServletRequest): Any = {
+ val logs =
+ if (hasAccess(session.owner, req)) {
+ Option(session.logLines())
+ .map { lines =>
+ val size = 10
+ var from = math.max(0, lines.length - size)
+ val until = from + size
+
+ lines.view(from, until)
+ }
+ .getOrElse(Nil)
+ } else {
+ Nil
+ }
+
+ new SessionInfo(session.id, session.appId.orNull, session.owner, session.proxyUser.orNull,
+ session.state.toString, session.kind.toString, session.appInfo.asJavaMap, logs.asJava)
+ }
+
+ post("/:id/stop") {
+ withSession { session =>
+ Await.ready(session.stop(), Duration.Inf)
+ NoContent()
+ }
+ }
+
+ post("/:id/interrupt") {
+ withSession { session =>
+ Await.ready(session.interrupt(), Duration.Inf)
+ Ok(Map("msg" -> "interrupted"))
+ }
+ }
+
+ get("/:id/statements") {
+ withSession { session =>
+ val statements = session.statements
+ val from = params.get("from").map(_.toInt).getOrElse(0)
+ val size = params.get("size").map(_.toInt).getOrElse(statements.length)
+
+ Map(
+ "total_statements" -> statements.length,
+ "statements" -> statements.view(from, from + size)
+ )
+ }
+ }
+
+ val getStatement = get("/:id/statements/:statementId") {
+ withSession { session =>
+ val statementId = params("statementId").toInt
+
+ session.getStatement(statementId).getOrElse(NotFound("Statement not found"))
+ }
+ }
+
+ jpost[ExecuteRequest]("/:id/statements") { req =>
+ withSession { session =>
+ val statement = session.executeStatement(req)
+
+ Created(statement,
+ headers = Map(
+ "Location" -> url(getStatement,
+ "id" -> session.id.toString,
+ "statementId" -> statement.id.toString)))
+ }
+ }
+
+ post("/:id/statements/:statementId/cancel") {
+ withSession { session =>
+ val statementId = params("statementId")
+ session.cancelStatement(statementId.toInt)
+ Ok(Map("msg" -> "canceled"))
+ }
+ }
+ // This endpoint is used by the client-http module to "connect" to an existing session and
+ // update its last activity time. It performs authorization checks to make sure the caller
+ // has access to the session, so even though it returns the same data, it behaves differently
+ // from get("/:id").
+ post("/:id/connect") {
+ withSession { session =>
+ session.recordActivity()
+ Ok(clientSessionView(session, request))
+ }
+ }
+
+ jpost[SerializedJob]("/:id/submit-job") { req =>
+ withSession { session =>
+ try {
+ require(req.job != null && req.job.length > 0, "no job provided.")
+ val jobId = session.submitJob(req.job)
+ Created(new JobStatus(jobId, JobHandle.State.SENT, null, null))
+ } catch {
+ case e: Throwable =>
+ e.printStackTrace()
+ throw e
+ }
+ }
+ }
+
+ jpost[SerializedJob]("/:id/run-job") { req =>
+ withSession { session =>
+ require(req.job != null && req.job.length > 0, "no job provided.")
+ val jobId = session.runJob(req.job)
+ Created(new JobStatus(jobId, JobHandle.State.SENT, null, null))
+ }
+ }
+
+ post("/:id/upload-jar") {
+ withSession { lsession =>
+ fileParams.get("jar") match {
+ case Some(file) =>
+ lsession.addJar(file.getInputStream, file.name)
+ case None =>
+ BadRequest("No jar sent!")
+ }
+ }
+ }
+
+ post("/:id/upload-pyfile") {
+ withSession { lsession =>
+ fileParams.get("file") match {
+ case Some(file) =>
+ lsession.addJar(file.getInputStream, file.name)
+ case None =>
+ BadRequest("No file sent!")
+ }
+ }
+ }
+
+ post("/:id/upload-file") {
+ withSession { lsession =>
+ fileParams.get("file") match {
+ case Some(file) =>
+ lsession.addFile(file.getInputStream, file.name)
+ case None =>
+ BadRequest("No file sent!")
+ }
+ }
+ }
+
+ jpost[AddResource]("/:id/add-jar") { req =>
+ withSession { lsession =>
+ addJarOrPyFile(req, lsession)
+ }
+ }
+
+ jpost[AddResource]("/:id/add-pyfile") { req =>
+ withSession { lsession =>
+ lsession.kind match {
+ case PySpark() | PySpark3() => addJarOrPyFile(req, lsession)
+ case _ => BadRequest("Only supported for pyspark sessions.")
+ }
+ }
+ }
+
+ jpost[AddResource]("/:id/add-file") { req =>
+ withSession { lsession =>
+ val uri = new URI(req.uri)
+ lsession.addFile(uri)
+ }
+ }
+
+ get("/:id/jobs/:jobid") {
+ withSession { lsession =>
+ val jobId = params("jobid").toLong
+ Ok(lsession.jobStatus(jobId))
+ }
+ }
+
+ post("/:id/jobs/:jobid/cancel") {
+ withSession { lsession =>
+ val jobId = params("jobid").toLong
+ lsession.cancelJob(jobId)
+ }
+ }
+
+ private def addJarOrPyFile(req: HttpMessages.AddResource, session: InteractiveSession): Unit = {
+ val uri = new URI(req.uri)
+ session.addJar(uri)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/interactive/SessionHeartbeat.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/interactive/SessionHeartbeat.scala b/server/src/main/scala/org/apache/livy/server/interactive/SessionHeartbeat.scala
new file mode 100644
index 0000000..b9a1fd7
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/interactive/SessionHeartbeat.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.server.interactive
+
+import java.util.Date
+
+import scala.concurrent.duration.{Deadline, Duration, FiniteDuration}
+
+import org.apache.livy.sessions.Session.RecoveryMetadata
+import org.apache.livy.LivyConf
+import org.apache.livy.server.SessionServlet
+import org.apache.livy.sessions.{Session, SessionManager}
+
+/**
+ * A session trait to provide heartbeat expiration check.
+ * Note: Session will not expire if heartbeat() was never called.
+ */
+trait SessionHeartbeat {
+ protected val heartbeatTimeout: FiniteDuration
+
+ private var _lastHeartbeat: Date = _ // For reporting purpose
+ private var heartbeatDeadline: Option[Deadline] = None
+
+ def heartbeat(): Unit = synchronized {
+ if (heartbeatTimeout > Duration.Zero) {
+ heartbeatDeadline = Some(heartbeatTimeout.fromNow)
+ }
+
+ _lastHeartbeat = new Date()
+ }
+
+ def lastHeartbeat: Date = synchronized { _lastHeartbeat }
+
+ def heartbeatExpired: Boolean = synchronized { heartbeatDeadline.exists(_.isOverdue()) }
+}
+
+/**
+ * Servlet can mixin this trait to update session's heartbeat
+ * whenever a /sessions/:id REST call is made. e.g. GET /sessions/:id
+ * Note: GET /sessions doesn't update heartbeats.
+ */
+trait SessionHeartbeatNotifier[S <: Session with SessionHeartbeat, R <: RecoveryMetadata]
+ extends SessionServlet[S, R] {
+
+ abstract override protected def withUnprotectedSession(fn: (S => Any)): Any = {
+ super.withUnprotectedSession { s =>
+ s.heartbeat()
+ fn(s)
+ }
+ }
+
+ abstract override protected def withSession(fn: (S => Any)): Any = {
+ super.withSession { s =>
+ s.heartbeat()
+ fn(s)
+ }
+ }
+}
+
+/**
+ * A SessionManager trait.
+ * It will create a thread that periodically deletes sessions with expired heartbeat.
+ */
+trait SessionHeartbeatWatchdog[S <: Session with SessionHeartbeat, R <: RecoveryMetadata] {
+ self: SessionManager[S, R] =>
+
+ private val watchdogThread = new Thread(s"HeartbeatWatchdog-${self.getClass.getName}") {
+ override def run(): Unit = {
+ val interval = livyConf.getTimeAsMs(LivyConf.HEARTBEAT_WATCHDOG_INTERVAL)
+ info("Heartbeat watchdog thread started.")
+ while (true) {
+ deleteExpiredSessions()
+ Thread.sleep(interval)
+ }
+ }
+ }
+
+ protected def start(): Unit = {
+ assert(!watchdogThread.isAlive())
+
+ watchdogThread.setDaemon(true)
+ watchdogThread.start()
+ }
+
+ private[interactive] def deleteExpiredSessions(): Unit = {
+ // Delete takes time. If we use .filter().foreach() here, the time difference between we check
+ // expiration and the time we delete the session might be huge. To avoid that, check expiration
+ // inside the foreach block.
+ sessions.values.foreach { s =>
+ if (s.heartbeatExpired) {
+ info(s"Session ${s.id} expired. Last heartbeat is at ${s.lastHeartbeat}.")
+ try { delete(s) } catch {
+ case t: Throwable =>
+ warn(s"Exception was thrown when deleting expired session ${s.id}", t)
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/recovery/BlackholeStateStore.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/recovery/BlackholeStateStore.scala b/server/src/main/scala/org/apache/livy/server/recovery/BlackholeStateStore.scala
new file mode 100644
index 0000000..df9a712
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/recovery/BlackholeStateStore.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.recovery
+
+import scala.reflect.ClassTag
+
+import org.apache.livy.LivyConf
+
+/**
+ * This is a blackhole implementation of StateStore.
+ * Livy will use this when session recovery is disabled.
+ */
+class BlackholeStateStore(livyConf: LivyConf) extends StateStore(livyConf) {
+ def set(key: String, value: Object): Unit = {}
+
+ def get[T: ClassTag](key: String): Option[T] = None
+
+ def getChildren(key: String): Seq[String] = List.empty[String]
+
+ def remove(key: String): Unit = {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/recovery/FileSystemStateStore.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/recovery/FileSystemStateStore.scala b/server/src/main/scala/org/apache/livy/server/recovery/FileSystemStateStore.scala
new file mode 100644
index 0000000..d5f8f3d
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/recovery/FileSystemStateStore.scala
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.recovery
+
+import java.io.{FileNotFoundException, IOException}
+import java.net.URI
+import java.util
+
+import scala.reflect.ClassTag
+import scala.util.control.NonFatal
+
+import org.apache.commons.io.IOUtils
+import org.apache.hadoop.fs._
+import org.apache.hadoop.fs.Options.{CreateOpts, Rename}
+import org.apache.hadoop.fs.permission.{FsAction, FsPermission}
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.Utils.usingResource
+
+class FileSystemStateStore(
+ livyConf: LivyConf,
+ mockFileContext: Option[FileContext])
+ extends StateStore(livyConf) with Logging {
+
+ // Constructor defined for StateStore factory to new this class using reflection.
+ def this(livyConf: LivyConf) {
+ this(livyConf, None)
+ }
+
+ private val fsUri = {
+ val fsPath = livyConf.get(LivyConf.RECOVERY_STATE_STORE_URL)
+ require(!fsPath.isEmpty, s"Please config ${LivyConf.RECOVERY_STATE_STORE_URL.key}.")
+ new URI(fsPath)
+ }
+
+ private val fileContext: FileContext = mockFileContext.getOrElse {
+ FileContext.getFileContext(fsUri)
+ }
+
+ {
+ // Only Livy user should have access to state files.
+ fileContext.setUMask(new FsPermission("077"))
+
+ // Create state store dir if it doesn't exist.
+ val stateStorePath = absPath(".")
+ try {
+ fileContext.mkdir(stateStorePath, FsPermission.getDirDefault(), true)
+ } catch {
+ case _: FileAlreadyExistsException =>
+ if (!fileContext.getFileStatus(stateStorePath).isDirectory()) {
+ throw new IOException(s"$stateStorePath is not a directory.")
+ }
+ }
+
+ // Check permission of state store dir.
+ val fileStatus = fileContext.getFileStatus(absPath("."))
+ require(fileStatus.getPermission.getUserAction() == FsAction.ALL,
+ s"Livy doesn't have permission to access state store: $fsUri.")
+ if (fileStatus.getPermission.getGroupAction != FsAction.NONE) {
+ warn(s"Group users have permission to access state store: $fsUri. This is insecure.")
+ }
+ if (fileStatus.getPermission.getOtherAction != FsAction.NONE) {
+ warn(s"Other users have permission to access state store: $fsUri. This is in secure.")
+ }
+ }
+
+ override def set(key: String, value: Object): Unit = {
+ // Write to a temp file then rename to avoid file corruption if livy-server crashes
+ // in the middle of the write.
+ val tmpPath = absPath(s"$key.tmp")
+ val createFlag = util.EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
+
+ usingResource(fileContext.create(tmpPath, createFlag, CreateOpts.createParent())) { tmpFile =>
+ tmpFile.write(serializeToBytes(value))
+ tmpFile.close()
+ // Assume rename is atomic.
+ fileContext.rename(tmpPath, absPath(key), Rename.OVERWRITE)
+ }
+
+ try {
+ val crcPath = new Path(tmpPath.getParent, s".${tmpPath.getName}.crc")
+ fileContext.delete(crcPath, false)
+ } catch {
+ case NonFatal(e) => // Swallow the exception.
+ }
+ }
+
+ override def get[T: ClassTag](key: String): Option[T] = {
+ try {
+ usingResource(fileContext.open(absPath(key))) { is =>
+ Option(deserialize[T](IOUtils.toByteArray(is)))
+ }
+ } catch {
+ case _: FileNotFoundException => None
+ case e: IOException =>
+ warn(s"Failed to read $key from state store.", e)
+ None
+ }
+ }
+
+ override def getChildren(key: String): Seq[String] = {
+ try {
+ fileContext.util.listStatus(absPath(key)).map(_.getPath.getName)
+ } catch {
+ case _: FileNotFoundException => Seq.empty
+ case e: IOException =>
+ warn(s"Failed to list $key from state store.", e)
+ Seq.empty
+ }
+ }
+
+ override def remove(key: String): Unit = {
+ fileContext.delete(absPath(key), false)
+ }
+
+ private def absPath(key: String): Path = new Path(fsUri.getPath(), key)
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/recovery/SessionStore.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/recovery/SessionStore.scala b/server/src/main/scala/org/apache/livy/server/recovery/SessionStore.scala
new file mode 100644
index 0000000..0429295
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/recovery/SessionStore.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.recovery
+
+import java.io.IOException
+
+import scala.reflect.ClassTag
+import scala.util.{Failure, Success, Try}
+import scala.util.control.NonFatal
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.sessions.Session.RecoveryMetadata
+
+private[recovery] case class SessionManagerState(nextSessionId: Int)
+
+/**
+ * SessionStore provides high level functions to get/save session state from/to StateStore.
+ */
+class SessionStore(
+ livyConf: LivyConf,
+ store: => StateStore = StateStore.get) // For unit testing.
+ extends Logging {
+
+ private val STORE_VERSION: String = "v1"
+
+ /**
+ * Persist a session to the session state store.
+ * @param m RecoveryMetadata for the session.
+ */
+ def save(sessionType: String, m: RecoveryMetadata): Unit = {
+ store.set(sessionPath(sessionType, m.id), m)
+ }
+
+ def saveNextSessionId(sessionType: String, id: Int): Unit = {
+ store.set(sessionManagerPath(sessionType), SessionManagerState(id))
+ }
+
+ /**
+ * Return all sessions stored in the store with specified session type.
+ */
+ def getAllSessions[T <: RecoveryMetadata : ClassTag](sessionType: String): Seq[Try[T]] = {
+ store.getChildren(sessionPath(sessionType))
+ .flatMap { c => Try(c.toInt).toOption } // Ignore all non numerical keys
+ .flatMap { id =>
+ val p = sessionPath(sessionType, id)
+ try {
+ store.get[T](p).map(Success(_))
+ } catch {
+ case NonFatal(e) => Some(Failure(new IOException(s"Error getting session $p", e)))
+ }
+ }
+ }
+
+ /**
+ * Return the next unused session id with specified session type.
+ * If checks the SessionManagerState stored and returns the next free session id.
+ * If no SessionManagerState is stored, it returns 0.
+ *
+ * @throws Exception If SessionManagerState stored is corrupted, it throws an error.
+ */
+ def getNextSessionId(sessionType: String): Int = {
+ store.get[SessionManagerState](sessionManagerPath(sessionType))
+ .map(_.nextSessionId).getOrElse(0)
+ }
+
+ /**
+ * Remove a session from the state store.
+ */
+ def remove(sessionType: String, id: Int): Unit = {
+ store.remove(sessionPath(sessionType, id))
+ }
+
+ private def sessionManagerPath(sessionType: String): String =
+ s"$STORE_VERSION/$sessionType/state"
+
+ private def sessionPath(sessionType: String): String =
+ s"$STORE_VERSION/$sessionType"
+
+ private def sessionPath(sessionType: String, id: Int): String =
+ s"$STORE_VERSION/$sessionType/$id"
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/recovery/StateStore.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/recovery/StateStore.scala b/server/src/main/scala/org/apache/livy/server/recovery/StateStore.scala
new file mode 100644
index 0000000..a6c3275
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/recovery/StateStore.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.server.recovery
+
+import scala.reflect.{classTag, ClassTag}
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.sessions.SessionKindModule
+import org.apache.livy.sessions.SessionManager._
+
+protected trait JsonMapper {
+ protected val mapper = new ObjectMapper()
+ .registerModule(DefaultScalaModule)
+ .registerModule(new SessionKindModule())
+
+ def serializeToBytes(value: Object): Array[Byte] = mapper.writeValueAsBytes(value)
+
+ def deserialize[T: ClassTag](json: Array[Byte]): T =
+ mapper.readValue(json, classTag[T].runtimeClass.asInstanceOf[Class[T]])
+}
+
+/**
+ * Interface of a key-value pair storage for state storage.
+ * It's responsible for de/serialization and retrieving/storing object.
+ * It's the low level interface used by higher level classes like SessionStore.
+ *
+ * Hardcoded to use JSON serialization for now for easier ops. Will add better serialization later.
+ */
+abstract class StateStore(livyConf: LivyConf) extends JsonMapper {
+ /**
+ * Set a key-value pair to this state store. It overwrites existing value.
+ * @throws Exception Throw when persisting the state store fails.
+ */
+ def set(key: String, value: Object): Unit
+
+ /**
+ * Get a key-value pair from this state store.
+ * @return Value if the key exists. None if the key doesn't exist.
+ * @throws Exception Throw when deserialization of the stored value fails.
+ */
+ def get[T: ClassTag](key: String): Option[T]
+
+ /**
+ * Treat keys in this state store as a directory tree and
+ * return names of the direct children of the key.
+ * @return List of names of the direct children of the key.
+ * Empty list if the key doesn't exist or have no child.
+ */
+ def getChildren(key: String): Seq[String]
+
+ /**
+ * Remove the key from this state store. Does not throw if the key doesn't exist.
+ * @throws Exception Throw when persisting the state store fails.
+ */
+ def remove(key: String): Unit
+}
+
+/**
+ * Factory to create the store chosen in LivyConf.
+ */
+object StateStore extends Logging {
+ private[this] var stateStore: Option[StateStore] = None
+
+ def init(livyConf: LivyConf): Unit = synchronized {
+ if (stateStore.isEmpty) {
+ val fileStateStoreClassTag = pickStateStore(livyConf)
+ stateStore = Option(fileStateStoreClassTag.getDeclaredConstructor(classOf[LivyConf])
+ .newInstance(livyConf).asInstanceOf[StateStore])
+ info(s"Using ${stateStore.get.getClass.getSimpleName} for recovery.")
+ }
+ }
+
+ def cleanup(): Unit = synchronized {
+ stateStore = None
+ }
+
+ def get: StateStore = {
+ assert(stateStore.isDefined, "StateStore hasn't been initialized.")
+ stateStore.get
+ }
+
+ private[recovery] def pickStateStore(livyConf: LivyConf): Class[_] = {
+ livyConf.get(LivyConf.RECOVERY_MODE) match {
+ case SESSION_RECOVERY_MODE_OFF => classOf[BlackholeStateStore]
+ case SESSION_RECOVERY_MODE_RECOVERY =>
+ livyConf.get(LivyConf.RECOVERY_STATE_STORE) match {
+ case "filesystem" => classOf[FileSystemStateStore]
+ case "zookeeper" => classOf[ZooKeeperStateStore]
+ case ss => throw new IllegalArgumentException(s"Unsupported state store $ss")
+ }
+ case rm => throw new IllegalArgumentException(s"Unsupported recovery mode $rm")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/recovery/ZooKeeperStateStore.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/recovery/ZooKeeperStateStore.scala b/server/src/main/scala/org/apache/livy/server/recovery/ZooKeeperStateStore.scala
new file mode 100644
index 0000000..ec6b9df
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/recovery/ZooKeeperStateStore.scala
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.server.recovery
+
+import scala.collection.JavaConverters._
+import scala.reflect.ClassTag
+import scala.util.Try
+
+import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
+import org.apache.curator.framework.api.UnhandledErrorListener
+import org.apache.curator.retry.RetryNTimes
+import org.apache.zookeeper.KeeperException.NoNodeException
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.LivyConf.Entry
+
+object ZooKeeperStateStore {
+ val ZK_KEY_PREFIX_CONF = Entry("livy.server.recovery.zk-state-store.key-prefix", "livy")
+ val ZK_RETRY_CONF = Entry("livy.server.recovery.zk-state-store.retry-policy", "5,100")
+}
+
+class ZooKeeperStateStore(
+ livyConf: LivyConf,
+ mockCuratorClient: Option[CuratorFramework] = None) // For testing
+ extends StateStore(livyConf) with Logging {
+
+ import ZooKeeperStateStore._
+
+ // Constructor defined for StateStore factory to new this class using reflection.
+ def this(livyConf: LivyConf) {
+ this(livyConf, None)
+ }
+
+ private val zkAddress = livyConf.get(LivyConf.RECOVERY_STATE_STORE_URL)
+ require(!zkAddress.isEmpty, s"Please config ${LivyConf.RECOVERY_STATE_STORE_URL.key}.")
+ private val zkKeyPrefix = livyConf.get(ZK_KEY_PREFIX_CONF)
+ private val retryValue = livyConf.get(ZK_RETRY_CONF)
+ // a regex to match patterns like "m, n" where m and n both are integer values
+ private val retryPattern = """\s*(\d+)\s*,\s*(\d+)\s*""".r
+ private[recovery] val retryPolicy = retryValue match {
+ case retryPattern(n, sleepMs) => new RetryNTimes(n.toInt, sleepMs.toInt)
+ case _ => throw new IllegalArgumentException(
+ s"$ZK_KEY_PREFIX_CONF contains bad value: $retryValue. " +
+ "Correct format is <max retry count>,<sleep ms between retry>. e.g. 5,100")
+ }
+
+ private val curatorClient = mockCuratorClient.getOrElse {
+ CuratorFrameworkFactory.newClient(zkAddress, retryPolicy)
+ }
+
+ Runtime.getRuntime.addShutdownHook(new Thread(new Runnable {
+ override def run(): Unit = {
+ curatorClient.close()
+ }
+ }))
+
+ curatorClient.getUnhandledErrorListenable().addListener(new UnhandledErrorListener {
+ def unhandledError(message: String, e: Throwable): Unit = {
+ error(s"Fatal Zookeeper error. Shutting down Livy server.")
+ System.exit(1)
+ }
+ })
+ curatorClient.start()
+ // TODO Make sure ZK path has proper secure permissions so that other users cannot read its
+ // contents.
+
+ override def set(key: String, value: Object): Unit = {
+ val prefixedKey = prefixKey(key)
+ val data = serializeToBytes(value)
+ if (curatorClient.checkExists().forPath(prefixedKey) == null) {
+ curatorClient.create().creatingParentsIfNeeded().forPath(prefixedKey, data)
+ } else {
+ curatorClient.setData().forPath(prefixedKey, data)
+ }
+ }
+
+ override def get[T: ClassTag](key: String): Option[T] = {
+ val prefixedKey = prefixKey(key)
+ if (curatorClient.checkExists().forPath(prefixedKey) == null) {
+ None
+ } else {
+ Option(deserialize[T](curatorClient.getData().forPath(prefixedKey)))
+ }
+ }
+
+ override def getChildren(key: String): Seq[String] = {
+ val prefixedKey = prefixKey(key)
+ if (curatorClient.checkExists().forPath(prefixedKey) == null) {
+ Seq.empty[String]
+ } else {
+ curatorClient.getChildren.forPath(prefixedKey).asScala
+ }
+ }
+
+ override def remove(key: String): Unit = {
+ try {
+ curatorClient.delete().guaranteed().forPath(prefixKey(key))
+ } catch {
+ case _: NoNodeException =>
+ }
+ }
+
+ private def prefixKey(key: String) = s"/$zkKeyPrefix/$key"
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala b/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala
new file mode 100644
index 0000000..7396bdc
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.ui
+
+import scala.xml.Node
+
+import org.scalatra.ScalatraServlet
+
+class UIServlet extends ScalatraServlet {
+ before() { contentType = "text/html" }
+
+ def getHeader(title: String): Seq[Node] =
+ <head>
+ <link rel="stylesheet" href="/static/bootstrap.min.css" type="text/css"/>
+ <link rel="stylesheet" href="/static/dataTables.bootstrap.min.css" type="text/css"/>
+ <link rel="stylesheet" href="/static/livy-ui.css" type="text/css"/>
+ <script src="/static/jquery-3.2.1.min.js"></script>
+ <script src="/static/bootstrap.min.js"></script>
+ <script src="/static/jquery.dataTables.min.js"></script>
+ <script src="/static/dataTables.bootstrap.min.js"></script>
+ <script src="/static/all-sessions.js"></script>
+ <title>{title}</title>
+ </head>
+
+ def navBar(pageName: String): Seq[Node] =
+ <nav class="navbar navbar-default">
+ <div class="container-fluid">
+ <div class="navbar-header">
+ <a class="navbar-brand" href="#">
+ <img alt="Livy" src="/static/livy-mini-logo.png"/>
+ </a>
+ </div>
+ <div class="collapse navbar-collapse">
+ <ul class="nav navbar-nav">
+ <li><a href="#">{pageName}</a></li>
+ </ul>
+ </div>
+ </div>
+ </nav>
+
+ def createPage(pageName: String, pageContents: Seq[Node]): Seq[Node] =
+ <html>
+ {getHeader("Livy - " + pageName)}
+ <body>
+ <div class="container">
+ {navBar(pageName)}
+ {pageContents}
+ </div>
+ </body>
+ </html>
+
+ get("/") {
+ val content =
+ <div id="all-sessions">
+ <div id="interactive-sessions"></div>
+ <div id="batches"></div>
+ </div>
+
+ createPage("Sessions", content)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/sessions/Session.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/sessions/Session.scala b/server/src/main/scala/org/apache/livy/sessions/Session.scala
new file mode 100644
index 0000000..d467076
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/sessions/Session.scala
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.sessions
+
+import java.io.InputStream
+import java.net.{URI, URISyntaxException}
+import java.security.PrivilegedExceptionAction
+import java.util.UUID
+import java.util.concurrent.TimeUnit
+
+import scala.concurrent.{ExecutionContext, Future}
+
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.hadoop.security.UserGroupInformation
+
+import org.apache.livy.{LivyConf, Logging, Utils}
+import org.apache.livy.utils.AppInfo
+
+object Session {
+ trait RecoveryMetadata { val id: Int }
+
+ lazy val configBlackList: Set[String] = {
+ val url = getClass.getResource("/spark-blacklist.conf")
+ if (url != null) Utils.loadProperties(url).keySet else Set()
+ }
+
+ /**
+ * Validates and prepares a user-provided configuration for submission.
+ *
+ * - Verifies that no blacklisted configurations are provided.
+ * - Merges file lists in the configuration with the explicit lists provided in the request
+ * - Resolve file URIs to make sure they reference the default FS
+ * - Verify that file URIs don't reference non-whitelisted local resources
+ */
+ def prepareConf(
+ conf: Map[String, String],
+ jars: Seq[String],
+ files: Seq[String],
+ archives: Seq[String],
+ pyFiles: Seq[String],
+ livyConf: LivyConf): Map[String, String] = {
+ if (conf == null) {
+ return Map()
+ }
+
+ val errors = conf.keySet.filter(configBlackList.contains)
+ if (errors.nonEmpty) {
+ throw new IllegalArgumentException(
+ "Blacklisted configuration values in session config: " + errors.mkString(", "))
+ }
+
+ val confLists: Map[String, Seq[String]] = livyConf.sparkFileLists
+ .map { key => (key -> Nil) }.toMap
+
+ val userLists = confLists ++ Map(
+ LivyConf.SPARK_JARS -> jars,
+ LivyConf.SPARK_FILES -> files,
+ LivyConf.SPARK_ARCHIVES -> archives,
+ LivyConf.SPARK_PY_FILES -> pyFiles)
+
+ val merged = userLists.flatMap { case (key, list) =>
+ val confList = conf.get(key)
+ .map { list =>
+ resolveURIs(list.split("[, ]+").toSeq, livyConf)
+ }
+ .getOrElse(Nil)
+ val userList = resolveURIs(list, livyConf)
+ if (confList.nonEmpty || userList.nonEmpty) {
+ Some(key -> (userList ++ confList).mkString(","))
+ } else {
+ None
+ }
+ }
+
+ val masterConfList = Map(LivyConf.SPARK_MASTER -> livyConf.sparkMaster()) ++
+ livyConf.sparkDeployMode().map(LivyConf.SPARK_DEPLOY_MODE -> _).toMap
+
+ conf ++ masterConfList ++ merged
+ }
+
+ /**
+ * Prepends the value of the "fs.defaultFS" configuration to any URIs that do not have a
+ * scheme. URIs are required to at least be absolute paths.
+ *
+ * @throws IllegalArgumentException If an invalid URI is found in the given list.
+ */
+ def resolveURIs(uris: Seq[String], livyConf: LivyConf): Seq[String] = {
+ val defaultFS = livyConf.hadoopConf.get("fs.defaultFS").stripSuffix("/")
+ uris.filter(_.nonEmpty).map { _uri =>
+ val uri = try {
+ new URI(_uri)
+ } catch {
+ case e: URISyntaxException => throw new IllegalArgumentException(e)
+ }
+ resolveURI(uri, livyConf).toString()
+ }
+ }
+
+ def resolveURI(uri: URI, livyConf: LivyConf): URI = {
+ val defaultFS = livyConf.hadoopConf.get("fs.defaultFS").stripSuffix("/")
+ val resolved =
+ if (uri.getScheme() == null) {
+ require(uri.getPath().startsWith("/"), s"Path '${uri.getPath()}' is not absolute.")
+ new URI(defaultFS + uri.getPath())
+ } else {
+ uri
+ }
+
+ if (resolved.getScheme() == "file") {
+ // Make sure the location is whitelisted before allowing local files to be added.
+ require(livyConf.localFsWhitelist.find(resolved.getPath().startsWith).isDefined,
+ s"Local path ${uri.getPath()} cannot be added to user sessions.")
+ }
+
+ resolved
+ }
+}
+
+abstract class Session(val id: Int, val owner: String, val livyConf: LivyConf)
+ extends Logging {
+
+ import Session._
+
+ protected implicit val executionContext = ExecutionContext.global
+
+ protected var _appId: Option[String] = None
+
+ private var _lastActivity = System.nanoTime()
+
+ // Directory where the session's staging files are created. The directory is only accessible
+ // to the session's effective user.
+ private var stagingDir: Path = null
+
+ def appId: Option[String] = _appId
+
+ var appInfo: AppInfo = AppInfo()
+
+ def lastActivity: Long = state match {
+ case SessionState.Error(time) => time
+ case SessionState.Dead(time) => time
+ case SessionState.Success(time) => time
+ case _ => _lastActivity
+ }
+
+ def logLines(): IndexedSeq[String]
+
+ def recordActivity(): Unit = {
+ _lastActivity = System.nanoTime()
+ }
+
+ def recoveryMetadata: RecoveryMetadata
+
+ def state: SessionState
+
+ def stop(): Future[Unit] = Future {
+ try {
+ info(s"Stopping $this...")
+ stopSession()
+ info(s"Stopped $this.")
+ } catch {
+ case e: Exception =>
+ warn(s"Error stopping session $id.", e)
+ }
+
+ try {
+ if (stagingDir != null) {
+ debug(s"Deleting session $id staging directory $stagingDir")
+ doAsOwner {
+ val fs = FileSystem.newInstance(livyConf.hadoopConf)
+ try {
+ fs.delete(stagingDir, true)
+ } finally {
+ fs.close()
+ }
+ }
+ }
+ } catch {
+ case e: Exception =>
+ warn(s"Error cleaning up session $id staging dir.", e)
+ }
+ }
+
+
+ override def toString(): String = s"${this.getClass.getSimpleName} $id"
+
+ protected def stopSession(): Unit
+
+ protected val proxyUser: Option[String]
+
+ protected def doAsOwner[T](fn: => T): T = {
+ val user = proxyUser.getOrElse(owner)
+ if (user != null) {
+ val ugi = if (UserGroupInformation.isSecurityEnabled) {
+ if (livyConf.getBoolean(LivyConf.IMPERSONATION_ENABLED)) {
+ UserGroupInformation.createProxyUser(user, UserGroupInformation.getCurrentUser())
+ } else {
+ UserGroupInformation.getCurrentUser()
+ }
+ } else {
+ UserGroupInformation.createRemoteUser(user)
+ }
+ ugi.doAs(new PrivilegedExceptionAction[T] {
+ override def run(): T = fn
+ })
+ } else {
+ fn
+ }
+ }
+
+ protected def copyResourceToHDFS(dataStream: InputStream, name: String): URI = doAsOwner {
+ val fs = FileSystem.newInstance(livyConf.hadoopConf)
+
+ try {
+ val filePath = new Path(getStagingDir(fs), name)
+ debug(s"Uploading user file to $filePath")
+
+ val outFile = fs.create(filePath, true)
+ val buffer = new Array[Byte](512 * 1024)
+ var read = -1
+ try {
+ while ({read = dataStream.read(buffer); read != -1}) {
+ outFile.write(buffer, 0, read)
+ }
+ } finally {
+ outFile.close()
+ }
+ filePath.toUri
+ } finally {
+ fs.close()
+ }
+ }
+
+ private def getStagingDir(fs: FileSystem): Path = synchronized {
+ if (stagingDir == null) {
+ val stagingRoot = Option(livyConf.get(LivyConf.SESSION_STAGING_DIR)).getOrElse {
+ new Path(fs.getHomeDirectory(), ".livy-sessions").toString()
+ }
+
+ val sessionDir = new Path(stagingRoot, UUID.randomUUID().toString())
+ fs.mkdirs(sessionDir)
+ fs.setPermission(sessionDir, new FsPermission("700"))
+ stagingDir = sessionDir
+ debug(s"Session $id staging directory is $stagingDir")
+ }
+ stagingDir
+ }
+
+}
[25/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/com/cloudera/livy/repl/ReplDriverSuite.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/com/cloudera/livy/repl/ReplDriverSuite.scala b/repl/src/test/scala/com/cloudera/livy/repl/ReplDriverSuite.scala
deleted file mode 100644
index c2eca91..0000000
--- a/repl/src/test/scala/com/cloudera/livy/repl/ReplDriverSuite.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import java.net.URI
-import java.util.concurrent.TimeUnit
-
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import org.apache.spark.launcher.SparkLauncher
-import org.json4s._
-import org.json4s.jackson.JsonMethods._
-import org.scalatest.FunSuite
-import org.scalatest.concurrent.Eventually._
-
-import com.cloudera.livy._
-import com.cloudera.livy.rsc.{PingJob, RSCClient, RSCConf}
-import com.cloudera.livy.sessions.Spark
-
-class ReplDriverSuite extends FunSuite with LivyBaseUnitTestSuite {
-
- private implicit val formats = DefaultFormats
-
- test("start a repl session using the rsc") {
- val client = new LivyClientBuilder()
- .setConf(SparkLauncher.DRIVER_MEMORY, "512m")
- .setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, sys.props("java.class.path"))
- .setConf(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH, sys.props("java.class.path"))
- .setConf(RSCConf.Entry.LIVY_JARS.key(), "")
- .setURI(new URI("rsc:/"))
- .setConf(RSCConf.Entry.DRIVER_CLASS.key(), classOf[ReplDriver].getName())
- .setConf(RSCConf.Entry.SESSION_KIND.key(), Spark().toString)
- .build()
- .asInstanceOf[RSCClient]
-
- try {
- // This is sort of what InteractiveSession.scala does to detect an idle session.
- client.submit(new PingJob()).get(60, TimeUnit.SECONDS)
-
- val statementId = client.submitReplCode("1 + 1").get
- eventually(timeout(30 seconds), interval(100 millis)) {
- val rawResult =
- client.getReplJobResults(statementId, 1).get(10, TimeUnit.SECONDS).statements(0)
- val result = rawResult.output
- assert((parse(result) \ Session.STATUS).extract[String] === Session.OK)
- }
- } finally {
- client.stop(true)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/com/cloudera/livy/repl/ScalaInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/com/cloudera/livy/repl/ScalaInterpreterSpec.scala b/repl/src/test/scala/com/cloudera/livy/repl/ScalaInterpreterSpec.scala
deleted file mode 100644
index 9df77c3..0000000
--- a/repl/src/test/scala/com/cloudera/livy/repl/ScalaInterpreterSpec.scala
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import org.apache.spark.SparkConf
-import org.json4s.{DefaultFormats, JValue}
-import org.json4s.JsonDSL._
-
-import com.cloudera.livy.rsc.RSCConf
-
-class ScalaInterpreterSpec extends BaseInterpreterSpec {
-
- implicit val formats = DefaultFormats
-
- override def createInterpreter(): Interpreter =
- new SparkInterpreter(new SparkConf())
-
- it should "execute `1 + 2` == 3" in withInterpreter { interpreter =>
- val response = interpreter.execute("1 + 2")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "res0: Int = 3"
- ))
- }
-
- it should "execute multiple statements" in withInterpreter { interpreter =>
- var response = interpreter.execute("val x = 1")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "x: Int = 1"
- ))
-
- response = interpreter.execute("val y = 2")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "y: Int = 2"
- ))
-
- response = interpreter.execute("x + y")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "res0: Int = 3"
- ))
- }
-
- it should "execute multiple statements in one block" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """
- |val x = 1
- |
- |val y = 2
- |
- |x + y
- """.stripMargin)
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "res2: Int = 3"
- ))
- }
-
- it should "do table magic" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """val x = List(List(1, "a"), List(3, "b"))
- |%table x
- """.stripMargin)
-
- response should equal(Interpreter.ExecuteSuccess(
- APPLICATION_LIVY_TABLE_JSON -> (
- ("headers" -> List(
- ("type" -> "BIGINT_TYPE") ~ ("name" -> "0"),
- ("type" -> "STRING_TYPE") ~ ("name" -> "1")
- )) ~
- ("data" -> List(
- List[JValue](1, "a"),
- List[JValue](3, "b")
- ))
- )
- ))
- }
-
- it should "allow magic inside statements" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """val x = List(List(1, "a"), List(3, "b"))
- |%table x
- |1 + 2
- """.stripMargin)
-
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "res0: Int = 3"
- ))
- }
-
- it should "capture stdout" in withInterpreter { interpreter =>
- val response = interpreter.execute("println(\"Hello World\")")
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "Hello World"
- ))
- }
-
- it should "report an error if accessing an unknown variable" in withInterpreter { interpreter =>
- interpreter.execute("x") match {
- case Interpreter.ExecuteError(ename, evalue, _) =>
- ename should equal ("Error")
- evalue should include ("error: not found: value x")
-
- case other =>
- fail(s"Expected error, got $other.")
- }
- }
-
- it should "execute spark commands" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """sc.parallelize(0 to 1).map { i => i+1 }.collect""".stripMargin)
-
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "res0: Array[Int] = Array(1, 2)"
- ))
- }
-
- it should "handle statements ending with comments" in withInterpreter { interpreter =>
- // Test statements with only comments
- var response = interpreter.execute("""// comment""")
- response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> ""))
-
- response = interpreter.execute(
- """/*
- |comment
- |*/
- """.stripMargin)
- response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> ""))
-
- // Test statements ending with comments
- response = interpreter.execute(
- """val r = 1
- |// comment
- """.stripMargin)
- response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> "r: Int = 1"))
-
- response = interpreter.execute(
- """val r = 1
- |/*
- |comment
- |comment
- |*/
- """.stripMargin)
- response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> "r: Int = 1"))
-
- // Test statements ending with a mix of single line and multi-line comments
- response = interpreter.execute(
- """val r = 1
- |// comment
- |/*
- |comment
- |comment
- |*/
- |// comment
- """.stripMargin)
- response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> "r: Int = 1"))
-
- response = interpreter.execute(
- """val r = 1
- |/*
- |comment
- |// comment
- |comment
- |*/
- """.stripMargin)
- response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> "r: Int = 1"))
-
- // Make sure incomplete statement is still returned as incomplete statement.
- response = interpreter.execute("sc.")
- response should equal(Interpreter.ExecuteIncomplete())
-
- // Make sure incomplete statement is still returned as incomplete statement.
- response = interpreter.execute(
- """sc.
- |// comment
- """.stripMargin)
- response should equal(Interpreter.ExecuteIncomplete())
-
- // Make sure our handling doesn't mess up a string with value like comments.
- val tripleQuotes = "\"\"\""
- val stringWithComment = s"/*\ncomment\n*/\n//comment"
- response = interpreter.execute(s"val r = $tripleQuotes$stringWithComment$tripleQuotes")
-
- try {
- response should equal(
- Interpreter.ExecuteSuccess(TEXT_PLAIN -> s"r: String = \n$stringWithComment"))
- } catch {
- case _: Exception =>
- response should equal(
- // Scala 2.11 doesn't have a " " after "="
- Interpreter.ExecuteSuccess(TEXT_PLAIN -> s"r: String =\n$stringWithComment"))
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/com/cloudera/livy/repl/SessionSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/com/cloudera/livy/repl/SessionSpec.scala b/repl/src/test/scala/com/cloudera/livy/repl/SessionSpec.scala
deleted file mode 100644
index 92d2322..0000000
--- a/repl/src/test/scala/com/cloudera/livy/repl/SessionSpec.scala
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import java.util.Properties
-import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit}
-
-import org.mockito.Mockito.when
-import org.mockito.invocation.InvocationOnMock
-import org.mockito.stubbing.Answer
-import org.scalatest.FunSpec
-import org.scalatest.Matchers._
-import org.scalatest.concurrent.Eventually
-import org.scalatest.mock.MockitoSugar.mock
-import org.scalatest.time._
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-import com.cloudera.livy.repl.Interpreter.ExecuteResponse
-import com.cloudera.livy.rsc.RSCConf
-
-class SessionSpec extends FunSpec with Eventually with LivyBaseUnitTestSuite {
- override implicit val patienceConfig =
- PatienceConfig(timeout = scaled(Span(10, Seconds)), interval = scaled(Span(100, Millis)))
-
- private val rscConf = new RSCConf(new Properties())
-
- describe("Session") {
- it("should call state changed callbacks in happy path") {
- val expectedStateTransitions =
- Array("not_started", "starting", "idle", "busy", "idle", "busy", "idle")
- val actualStateTransitions = new ConcurrentLinkedQueue[String]()
-
- val interpreter = mock[Interpreter]
- when(interpreter.kind).thenAnswer(new Answer[String] {
- override def answer(invocationOnMock: InvocationOnMock): String = "spark"
- })
-
- val session =
- new Session(rscConf, interpreter, { s => actualStateTransitions.add(s.toString) })
-
- session.start()
-
- session.execute("")
-
- eventually {
- actualStateTransitions.toArray shouldBe expectedStateTransitions
- }
- }
-
- it("should not transit to idle if there're any pending statements.") {
- val expectedStateTransitions =
- Array("not_started", "busy", "busy", "busy", "idle", "busy", "idle")
- val actualStateTransitions = new ConcurrentLinkedQueue[String]()
-
- val interpreter = mock[Interpreter]
- when(interpreter.kind).thenAnswer(new Answer[String] {
- override def answer(invocationOnMock: InvocationOnMock): String = "spark"
- })
-
- val blockFirstExecuteCall = new CountDownLatch(1)
- when(interpreter.execute("")).thenAnswer(new Answer[Interpreter.ExecuteResponse] {
- override def answer(invocation: InvocationOnMock): ExecuteResponse = {
- blockFirstExecuteCall.await(10, TimeUnit.SECONDS)
- null
- }
- })
- val session =
- new Session(rscConf, interpreter, { s => actualStateTransitions.add(s.toString) })
-
- for (_ <- 1 to 2) {
- session.execute("")
- }
-
- blockFirstExecuteCall.countDown()
- eventually {
- actualStateTransitions.toArray shouldBe expectedStateTransitions
- }
- }
-
- it("should remove old statements when reaching threshold") {
- val interpreter = mock[Interpreter]
- when(interpreter.kind).thenAnswer(new Answer[String] {
- override def answer(invocationOnMock: InvocationOnMock): String = "spark"
- })
-
- rscConf.set(RSCConf.Entry.RETAINED_STATEMENT_NUMBER, 2)
- val session = new Session(rscConf, interpreter)
- session.start()
-
- session.statements.size should be (0)
- session.execute("")
- session.statements.size should be (1)
- session.statements.map(_._1).toSet should be (Set(0))
- session.execute("")
- session.statements.size should be (2)
- session.statements.map(_._1).toSet should be (Set(0, 1))
- session.execute("")
- eventually {
- session.statements.size should be (2)
- session.statements.map(_._1).toSet should be (Set(1, 2))
- }
-
- // Continue submitting statements, total statements in memory should be 2.
- session.execute("")
- eventually {
- session.statements.size should be (2)
- session.statements.map(_._1).toSet should be (Set(2, 3))
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/com/cloudera/livy/repl/SparkRInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/com/cloudera/livy/repl/SparkRInterpreterSpec.scala b/repl/src/test/scala/com/cloudera/livy/repl/SparkRInterpreterSpec.scala
deleted file mode 100644
index bde49b1..0000000
--- a/repl/src/test/scala/com/cloudera/livy/repl/SparkRInterpreterSpec.scala
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import org.apache.spark.SparkConf
-import org.json4s.{DefaultFormats, JValue}
-import org.json4s.JsonDSL._
-import org.scalatest._
-
-import com.cloudera.livy.rsc.RSCConf
-
-class SparkRInterpreterSpec extends BaseInterpreterSpec {
-
- implicit val formats = DefaultFormats
-
- override protected def withFixture(test: NoArgTest): Outcome = {
- assume(!sys.props.getOrElse("skipRTests", "false").toBoolean, "Skipping R tests.")
- super.withFixture(test)
- }
-
- override def createInterpreter(): Interpreter = SparkRInterpreter(new SparkConf())
-
- it should "execute `1 + 2` == 3" in withInterpreter { interpreter =>
- val response = interpreter.execute("1 + 2")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "[1] 3"
- ))
- }
-
- it should "execute multiple statements" in withInterpreter { interpreter =>
- var response = interpreter.execute("x = 1")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> ""
- ))
-
- response = interpreter.execute("y = 2")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> ""
- ))
-
- response = interpreter.execute("x + y")
- response should equal (Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "[1] 3"
- ))
- }
-
- it should "execute multiple statements in one block" in withInterpreter { interpreter =>
- val response = interpreter.execute(
- """
- |x = 1
- |
- |y = 2
- |
- |x + y
- """.stripMargin)
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "[1] 3"
- ))
- }
-
- it should "capture stdout" in withInterpreter { interpreter =>
- val response = interpreter.execute("cat(3)")
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "3"
- ))
- }
-
- it should "report an error if accessing an unknown variable" in withInterpreter { interpreter =>
- val response = interpreter.execute("x")
- assert(response.isInstanceOf[Interpreter.ExecuteError])
- val errorResponse = response.asInstanceOf[Interpreter.ExecuteError]
- errorResponse.ename should be ("Error")
- assert(errorResponse.evalue.contains("object 'x' not found"))
- }
-
-
- it should "not hang when executing incomplete statements" in withInterpreter { interpreter =>
- val response = interpreter.execute("x[")
- response should equal(Interpreter.ExecuteError(
- "Error",
- """[1] "Error in parse(text = \"x[\"): <text>:2:0: unexpected end of input\n1: x[\n ^""""
- ))
- }
-
- it should "escape the statement" in withInterpreter { interpreter =>
- val response = interpreter.execute("print(\"a\")")
- response should equal(Interpreter.ExecuteSuccess(
- TEXT_PLAIN -> "[1] \"a\""
- ))
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/com/cloudera/livy/repl/SparkRSessionSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/com/cloudera/livy/repl/SparkRSessionSpec.scala b/repl/src/test/scala/com/cloudera/livy/repl/SparkRSessionSpec.scala
deleted file mode 100644
index 2ca4130..0000000
--- a/repl/src/test/scala/com/cloudera/livy/repl/SparkRSessionSpec.scala
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import org.apache.spark.SparkConf
-import org.json4s.Extraction
-import org.json4s.jackson.JsonMethods.parse
-
-import com.cloudera.livy.rsc.RSCConf
-
-class SparkRSessionSpec extends BaseSessionSpec {
-
- override protected def withFixture(test: NoArgTest) = {
- assume(!sys.props.getOrElse("skipRTests", "false").toBoolean, "Skipping R tests.")
- super.withFixture(test)
- }
-
- override def createInterpreter(): Interpreter = SparkRInterpreter(new SparkConf())
-
- it should "execute `1 + 2` == 3" in withSession { session =>
- val statement = execute(session)("1 + 2")
- statement.id should equal(0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "[1] 3"
- )
- ))
-
- result should equal(expectedResult)
- }
-
- it should "execute `x = 1`, then `y = 2`, then `x + y`" in withSession { session =>
- val executeWithSession = execute(session)(_)
- var statement = executeWithSession("x = 1")
- statement.id should equal (0)
-
- var result = parse(statement.output)
- var expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> ""
- )
- ))
-
- result should equal (expectedResult)
-
- statement = executeWithSession("y = 2")
- statement.id should equal (1)
-
- result = parse(statement.output)
- expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 1,
- "data" -> Map(
- "text/plain" -> ""
- )
- ))
-
- result should equal (expectedResult)
-
- statement = executeWithSession("x + y")
- statement.id should equal (2)
-
- result = parse(statement.output)
- expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 2,
- "data" -> Map(
- "text/plain" -> "[1] 3"
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "capture stdout from print" in withSession { session =>
- val statement = execute(session)("""print('Hello World')""")
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "[1] \"Hello World\""
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "capture stdout from cat" in withSession { session =>
- val statement = execute(session)("""cat(3)""")
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "3"
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "report an error if accessing an unknown variable" in withSession { session =>
- val statement = execute(session)("""x""")
- statement.id should equal (0)
-
- val result = parse(statement.output)
- (result \ "status").extract[String] should be ("error")
- (result \ "execution_count").extract[Int] should be (0)
- (result \ "ename").extract[String] should be ("Error")
- assert((result \ "evalue").extract[String].contains("object 'x' not found"))
- (result \ "traceback").extract[List[String]] should be (List())
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/com/cloudera/livy/repl/SparkSessionSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/com/cloudera/livy/repl/SparkSessionSpec.scala b/repl/src/test/scala/com/cloudera/livy/repl/SparkSessionSpec.scala
deleted file mode 100644
index 1b2be30..0000000
--- a/repl/src/test/scala/com/cloudera/livy/repl/SparkSessionSpec.scala
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import org.apache.spark.SparkConf
-import org.json4s.Extraction
-import org.json4s.JsonAST.JValue
-import org.json4s.jackson.JsonMethods.parse
-import org.scalatest.concurrent.Eventually._
-
-import com.cloudera.livy.rsc.RSCConf
-import com.cloudera.livy.rsc.driver.StatementState
-
-class SparkSessionSpec extends BaseSessionSpec {
-
- override def createInterpreter(): Interpreter = new SparkInterpreter(new SparkConf())
-
- it should "execute `1 + 2` == 3" in withSession { session =>
- val statement = execute(session)("1 + 2")
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "res0: Int = 3"
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "execute `x = 1`, then `y = 2`, then `x + y`" in withSession { session =>
- val executeWithSession = execute(session)(_)
- var statement = executeWithSession("val x = 1")
- statement.id should equal (0)
-
- var result = parse(statement.output)
- var expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "x: Int = 1"
- )
- ))
-
- result should equal (expectedResult)
-
- statement = executeWithSession("val y = 2")
- statement.id should equal (1)
-
- result = parse(statement.output)
- expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 1,
- "data" -> Map(
- "text/plain" -> "y: Int = 2"
- )
- ))
-
- result should equal (expectedResult)
-
- statement = executeWithSession("x + y")
- statement.id should equal (2)
-
- result = parse(statement.output)
- expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 2,
- "data" -> Map(
- "text/plain" -> "res0: Int = 3"
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "capture stdout" in withSession { session =>
- val statement = execute(session)("""println("Hello World")""")
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "Hello World"
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "report an error if accessing an unknown variable" in withSession { session =>
- val statement = execute(session)("""x""")
- statement.id should equal (0)
-
- val result = parse(statement.output)
-
- def extract(key: String): String = (result \ key).extract[String]
-
- extract("status") should equal ("error")
- extract("execution_count") should equal ("0")
- extract("ename") should equal ("Error")
- extract("evalue") should include ("error: not found: value x")
- }
-
- it should "report an error if exception is thrown" in withSession { session =>
- val statement = execute(session)(
- """def func1() {
- |throw new Exception()
- |}
- |func1()""".stripMargin)
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val resultMap = result.extract[Map[String, JValue]]
-
- // Manually extract the values since the line numbers in the exception could change.
- resultMap("status").extract[String] should equal ("error")
- resultMap("execution_count").extract[Int] should equal (0)
- resultMap("ename").extract[String] should equal ("Error")
- resultMap("evalue").extract[String] should include ("java.lang.Exception")
-
- val traceback = resultMap("traceback").extract[Seq[String]]
- traceback(0) should include ("func1(<console>:")
- }
-
- it should "access the spark context" in withSession { session =>
- val statement = execute(session)("""sc""")
- statement.id should equal (0)
-
- val result = parse(statement.output)
- val resultMap = result.extract[Map[String, JValue]]
-
- // Manually extract the values since the line numbers in the exception could change.
- resultMap("status").extract[String] should equal ("ok")
- resultMap("execution_count").extract[Int] should equal (0)
-
- val data = resultMap("data").extract[Map[String, JValue]]
- data("text/plain").extract[String] should include (
- "res0: org.apache.spark.SparkContext = org.apache.spark.SparkContext")
- }
-
- it should "execute spark commands" in withSession { session =>
- val statement = execute(session)(
- """sc.parallelize(0 to 1).map{i => i+1}.collect""".stripMargin)
- statement.id should equal (0)
-
- val result = parse(statement.output)
-
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "res0: Array[Int] = Array(1, 2)"
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "do table magic" in withSession { session =>
- val statement = execute(session)("val x = List((1, \"a\"), (3, \"b\"))\n%table x")
- statement.id should equal (0)
-
- val result = parse(statement.output)
-
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "application/vnd.livy.table.v1+json" -> Map(
- "headers" -> List(
- Map("type" -> "BIGINT_TYPE", "name" -> "_1"),
- Map("type" -> "STRING_TYPE", "name" -> "_2")),
- "data" -> List(List(1, "a"), List(3, "b"))
- )
- )
- ))
-
- result should equal (expectedResult)
- }
-
- it should "cancel spark jobs" in withSession { session =>
- val stmtId = session.execute(
- """sc.parallelize(0 to 10).map { i => Thread.sleep(10000); i + 1 }.collect""".stripMargin)
- eventually(timeout(30 seconds), interval(100 millis)) {
- assert(session.statements(stmtId).state.get() == StatementState.Running)
- }
- session.cancel(stmtId)
-
- eventually(timeout(30 seconds), interval(100 millis)) {
- assert(session.statements(stmtId).state.get() == StatementState.Cancelled)
- session.statements(stmtId).output should include (
- "Job 0 cancelled part of cancelled job group 0")
- }
- }
-
- it should "cancel waiting statement" in withSession { session =>
- val stmtId1 = session.execute(
- """sc.parallelize(0 to 10).map { i => Thread.sleep(10000); i + 1 }.collect""".stripMargin)
- val stmtId2 = session.execute(
- """sc.parallelize(0 to 10).map { i => Thread.sleep(10000); i + 1 }.collect""".stripMargin)
- eventually(timeout(30 seconds), interval(100 millis)) {
- assert(session.statements(stmtId1).state.get() == StatementState.Running)
- }
-
- assert(session.statements(stmtId2).state.get() == StatementState.Waiting)
-
- session.cancel(stmtId2)
- assert(session.statements(stmtId2).state.get() == StatementState.Cancelled)
-
- session.cancel(stmtId1)
- assert(session.statements(stmtId1).state.get() == StatementState.Cancelling)
- eventually(timeout(30 seconds), interval(100 millis)) {
- assert(session.statements(stmtId1).state.get() == StatementState.Cancelled)
- session.statements(stmtId1).output should include (
- "Job 0 cancelled part of cancelled job group 0")
- }
- }
-
- it should "correctly calculate progress" in withSession { session =>
- val executeCode =
- """
- |sc.parallelize(1 to 2, 2).map(i => (i, 1)).collect()
- """.stripMargin
-
- val stmtId = session.execute(executeCode)
- eventually(timeout(30 seconds), interval(100 millis)) {
- session.progressOfStatement(stmtId) should be(1.0)
- }
- }
-
- it should "not generate Spark jobs for plain Scala code" in withSession { session =>
- val executeCode = """1 + 1"""
-
- val stmtId = session.execute(executeCode)
- session.progressOfStatement(stmtId) should be (0.0)
- }
-
- it should "handle multiple jobs in one statement" in withSession { session =>
- val executeCode =
- """
- |sc.parallelize(1 to 2, 2).map(i => (i, 1)).collect()
- |sc.parallelize(1 to 2, 2).map(i => (i, 1)).collect()
- """.stripMargin
-
- val stmtId = session.execute(executeCode)
- eventually(timeout(30 seconds), interval(100 millis)) {
- session.progressOfStatement(stmtId) should be(1.0)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/org/apache/livy/repl/BaseInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/livy/repl/BaseInterpreterSpec.scala b/repl/src/test/scala/org/apache/livy/repl/BaseInterpreterSpec.scala
new file mode 100644
index 0000000..b3d4d5a
--- /dev/null
+++ b/repl/src/test/scala/org/apache/livy/repl/BaseInterpreterSpec.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import org.scalatest.{FlatSpec, Matchers}
+
+import org.apache.livy.LivyBaseUnitTestSuite
+
+abstract class BaseInterpreterSpec extends FlatSpec with Matchers with LivyBaseUnitTestSuite {
+
+ def createInterpreter(): Interpreter
+
+ def withInterpreter(testCode: Interpreter => Any): Unit = {
+ val interpreter = createInterpreter()
+ try {
+ interpreter.start()
+ testCode(interpreter)
+ } finally {
+ interpreter.close()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/org/apache/livy/repl/BaseSessionSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/livy/repl/BaseSessionSpec.scala b/repl/src/test/scala/org/apache/livy/repl/BaseSessionSpec.scala
new file mode 100644
index 0000000..a13f924
--- /dev/null
+++ b/repl/src/test/scala/org/apache/livy/repl/BaseSessionSpec.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import java.util.Properties
+import java.util.concurrent.atomic.AtomicInteger
+
+import scala.concurrent.Await
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.json4s._
+import org.scalatest.{FlatSpec, Matchers}
+import org.scalatest.concurrent.Eventually._
+
+import org.apache.livy.LivyBaseUnitTestSuite
+import org.apache.livy.rsc.RSCConf
+import org.apache.livy.rsc.driver.{Statement, StatementState}
+import org.apache.livy.sessions.SessionState
+
+abstract class BaseSessionSpec extends FlatSpec with Matchers with LivyBaseUnitTestSuite {
+
+ implicit val formats = DefaultFormats
+
+ private val rscConf = new RSCConf(new Properties())
+
+ protected def execute(session: Session)(code: String): Statement = {
+ val id = session.execute(code)
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ val s = session.statements(id)
+ s.state.get() shouldBe StatementState.Available
+ s
+ }
+ }
+
+ protected def withSession(testCode: Session => Any): Unit = {
+ val stateChangedCalled = new AtomicInteger()
+ val session =
+ new Session(rscConf, createInterpreter(), { _ => stateChangedCalled.incrementAndGet() })
+ try {
+ // Session's constructor should fire an initial state change event.
+ stateChangedCalled.intValue() shouldBe 1
+ Await.ready(session.start(), 30 seconds)
+ assert(session.state === SessionState.Idle())
+ // There should be at least 1 state change event fired when session transits to idle.
+ stateChangedCalled.intValue() should (be > 1)
+ testCode(session)
+ } finally {
+ session.close()
+ }
+ }
+
+ protected def createInterpreter(): Interpreter
+
+ it should "start in the starting or idle state" in {
+ val session = new Session(rscConf, createInterpreter())
+ val future = session.start()
+ try {
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ session.state should (equal (SessionState.Starting()) or equal (SessionState.Idle()))
+ }
+ Await.ready(future, 60 seconds)
+ } finally {
+ session.close()
+ }
+ }
+
+ it should "eventually become the idle state" in withSession { session =>
+ session.state should equal (SessionState.Idle())
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/org/apache/livy/repl/PythonInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/livy/repl/PythonInterpreterSpec.scala b/repl/src/test/scala/org/apache/livy/repl/PythonInterpreterSpec.scala
new file mode 100644
index 0000000..1404765
--- /dev/null
+++ b/repl/src/test/scala/org/apache/livy/repl/PythonInterpreterSpec.scala
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import org.apache.spark.SparkConf
+import org.json4s.{DefaultFormats, JNull, JValue}
+import org.json4s.JsonDSL._
+import org.scalatest._
+
+import org.apache.livy.rsc.RSCConf
+import org.apache.livy.sessions._
+
+abstract class PythonBaseInterpreterSpec extends BaseInterpreterSpec {
+
+ it should "execute `1 + 2` == 3" in withInterpreter { interpreter =>
+ val response = interpreter.execute("1 + 2")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "3"
+ ))
+ }
+
+ it should "execute multiple statements" in withInterpreter { interpreter =>
+ var response = interpreter.execute("x = 1")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> ""
+ ))
+
+ response = interpreter.execute("y = 2")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> ""
+ ))
+
+ response = interpreter.execute("x + y")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "3"
+ ))
+ }
+
+ it should "execute multiple statements in one block" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """
+ |x = 1
+ |
+ |y = 2
+ |
+ |x + y
+ """.stripMargin)
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "3"
+ ))
+ }
+
+ it should "parse a class" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """
+ |class Counter(object):
+ | def __init__(self):
+ | self.count = 0
+ |
+ | def add_one(self):
+ | self.count += 1
+ |
+ | def add_two(self):
+ | self.count += 2
+ |
+ |counter = Counter()
+ |counter.add_one()
+ |counter.add_two()
+ |counter.count
+ """.stripMargin)
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "3"
+ ))
+ }
+
+ it should "do json magic" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """x = [[1, 'a'], [3, 'b']]
+ |%json x
+ """.stripMargin)
+
+ response should equal(Interpreter.ExecuteSuccess(
+ APPLICATION_JSON -> List[JValue](
+ List[JValue](1, "a"),
+ List[JValue](3, "b")
+ )
+ ))
+ }
+
+ it should "do table magic" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """x = [[1, 'a'], [3, 'b']]
+ |%table x
+ """.stripMargin)
+
+ response should equal(Interpreter.ExecuteSuccess(
+ APPLICATION_LIVY_TABLE_JSON -> (
+ ("headers" -> List(
+ ("type" -> "INT_TYPE") ~ ("name" -> "0"),
+ ("type" -> "STRING_TYPE") ~ ("name" -> "1")
+ )) ~
+ ("data" -> List(
+ List[JValue](1, "a"),
+ List[JValue](3, "b")
+ ))
+ )
+ ))
+ }
+
+ it should "do table magic with None type value" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """x = [{"a":"1", "b":None}, {"a":"2", "b":2}]
+ |%table x
+ """.stripMargin)
+
+ response should equal(Interpreter.ExecuteSuccess(
+ APPLICATION_LIVY_TABLE_JSON -> (
+ ("headers" -> List(
+ ("type" -> "STRING_TYPE") ~ ("name" -> "a"),
+ ("type" -> "INT_TYPE") ~ ("name" -> "b")
+ )) ~
+ ("data" -> List(
+ List[JValue]("1", JNull),
+ List[JValue]("2", 2)
+ ))
+ )
+ ))
+ }
+
+ it should "do table magic with None type Row" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """x = [{"a":None, "b":None}, {"a":"2", "b":2}]
+ |%table x
+ """.stripMargin)
+
+ response should equal(Interpreter.ExecuteSuccess(
+ APPLICATION_LIVY_TABLE_JSON -> (
+ ("headers" -> List(
+ ("type" -> "STRING_TYPE") ~ ("name" -> "a"),
+ ("type" -> "INT_TYPE") ~ ("name" -> "b")
+ )) ~
+ ("data" -> List(
+ List[JValue](JNull, JNull),
+ List[JValue]("2", 2)
+ ))
+ )
+ ))
+ }
+
+ it should "allow magic inside statements" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """x = [[1, 'a'], [3, 'b']]
+ |%table x
+ |1 + 2
+ """.stripMargin)
+
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "3"
+ ))
+ }
+
+ it should "capture stdout" in withInterpreter { interpreter =>
+ val response = interpreter.execute("print('Hello World')")
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "Hello World"
+ ))
+ }
+
+ it should "report an error if accessing an unknown variable" in withInterpreter { interpreter =>
+ val response = interpreter.execute("x")
+ response should equal(Interpreter.ExecuteError(
+ "NameError",
+ "name 'x' is not defined",
+ List(
+ "Traceback (most recent call last):\n",
+ "NameError: name 'x' is not defined\n"
+ )
+ ))
+ }
+
+ it should "report an error if empty magic command" in withInterpreter { interpreter =>
+ val response = interpreter.execute("%")
+ response should equal(Interpreter.ExecuteError(
+ "UnknownMagic",
+ "magic command not specified",
+ List("UnknownMagic: magic command not specified\n")
+ ))
+ }
+
+ it should "report an error if unknown magic command" in withInterpreter { interpreter =>
+ val response = interpreter.execute("%foo")
+ response should equal(Interpreter.ExecuteError(
+ "UnknownMagic",
+ "unknown magic command 'foo'",
+ List("UnknownMagic: unknown magic command 'foo'\n")
+ ))
+ }
+
+ it should "not execute part of the block if there is a syntax error" in withInterpreter { intp =>
+ var response = intp.execute(
+ """x = 1
+ |'
+ """.stripMargin)
+
+ response should equal(Interpreter.ExecuteError(
+ "SyntaxError",
+ "EOL while scanning string literal (<stdin>, line 2)",
+ List(
+ " File \"<stdin>\", line 2\n",
+ " '\n",
+ " ^\n",
+ "SyntaxError: EOL while scanning string literal\n"
+ )
+ ))
+
+ response = intp.execute("x")
+ response should equal(Interpreter.ExecuteError(
+ "NameError",
+ "name 'x' is not defined",
+ List(
+ "Traceback (most recent call last):\n",
+ "NameError: name 'x' is not defined\n"
+ )
+ ))
+ }
+}
+
+class Python2InterpreterSpec extends PythonBaseInterpreterSpec {
+
+ implicit val formats = DefaultFormats
+
+ override def createInterpreter(): Interpreter = PythonInterpreter(new SparkConf(), PySpark())
+
+ // Scalastyle is treating unicode escape as non ascii characters. Turn off the check.
+ // scalastyle:off non.ascii.character.disallowed
+ it should "print unicode correctly" in withInterpreter { intp =>
+ intp.execute("print(u\"\u263A\")") should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "\u263A"
+ ))
+ intp.execute("""print(u"\u263A")""") should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "\u263A"
+ ))
+ intp.execute("""print("\xE2\x98\xBA")""") should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "\u263A"
+ ))
+ }
+ // scalastyle:on non.ascii.character.disallowed
+}
+
+class Python3InterpreterSpec extends PythonBaseInterpreterSpec {
+
+ implicit val formats = DefaultFormats
+
+ override protected def withFixture(test: NoArgTest): Outcome = {
+ assume(!sys.props.getOrElse("skipPySpark3Tests", "false").toBoolean, "Skipping PySpark3 tests.")
+ test()
+ }
+
+ override def createInterpreter(): Interpreter = PythonInterpreter(new SparkConf(), PySpark3())
+
+ it should "check python version is 3.x" in withInterpreter { interpreter =>
+ val response = interpreter.execute("""import sys
+ |sys.version >= '3'
+ """.stripMargin)
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "True"
+ ))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/org/apache/livy/repl/PythonSessionSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/livy/repl/PythonSessionSpec.scala b/repl/src/test/scala/org/apache/livy/repl/PythonSessionSpec.scala
new file mode 100644
index 0000000..0883f27
--- /dev/null
+++ b/repl/src/test/scala/org/apache/livy/repl/PythonSessionSpec.scala
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import org.apache.spark.SparkConf
+import org.json4s.Extraction
+import org.json4s.jackson.JsonMethods.parse
+import org.scalatest._
+
+import org.apache.livy.rsc.RSCConf
+import org.apache.livy.sessions._
+
+abstract class PythonSessionSpec extends BaseSessionSpec {
+
+ it should "execute `1 + 2` == 3" in withSession { session =>
+ val statement = execute(session)("1 + 2")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "3"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "execute `x = 1`, then `y = 2`, then `x + y`" in withSession { session =>
+ val executeWithSession = execute(session)(_)
+ var statement = executeWithSession("x = 1")
+ statement.id should equal (0)
+
+ var result = parse(statement.output)
+ var expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> ""
+ )
+ ))
+
+ result should equal (expectedResult)
+
+ statement = executeWithSession("y = 2")
+ statement.id should equal (1)
+
+ result = parse(statement.output)
+ expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 1,
+ "data" -> Map(
+ "text/plain" -> ""
+ )
+ ))
+
+ result should equal (expectedResult)
+
+ statement = executeWithSession("x + y")
+ statement.id should equal (2)
+
+ result = parse(statement.output)
+ expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 2,
+ "data" -> Map(
+ "text/plain" -> "3"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "do table magic" in withSession { session =>
+ val statement = execute(session)("x = [[1, 'a'], [3, 'b']]\n%table x")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "application/vnd.livy.table.v1+json" -> Map(
+ "headers" -> List(
+ Map("type" -> "INT_TYPE", "name" -> "0"),
+ Map("type" -> "STRING_TYPE", "name" -> "1")),
+ "data" -> List(List(1, "a"), List(3, "b"))
+ )
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "capture stdout" in withSession { session =>
+ val statement = execute(session)("""print('Hello World')""")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "Hello World"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "report an error if accessing an unknown variable" in withSession { session =>
+ val statement = execute(session)("""x""")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "error",
+ "execution_count" -> 0,
+ "traceback" -> List(
+ "Traceback (most recent call last):\n",
+ "NameError: name 'x' is not defined\n"
+ ),
+ "ename" -> "NameError",
+ "evalue" -> "name 'x' is not defined"
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "report an error if exception is thrown" in withSession { session =>
+ val statement = execute(session)(
+ """def func1():
+ | raise Exception("message")
+ |def func2():
+ | func1()
+ |func2()
+ """.stripMargin)
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "error",
+ "execution_count" -> 0,
+ "traceback" -> List(
+ "Traceback (most recent call last):\n",
+ " File \"<stdin>\", line 4, in func2\n",
+ " File \"<stdin>\", line 2, in func1\n",
+ "Exception: message\n"
+ ),
+ "ename" -> "Exception",
+ "evalue" -> "message"
+ ))
+
+ result should equal (expectedResult)
+ }
+}
+
+class Python2SessionSpec extends PythonSessionSpec {
+ override def createInterpreter(): Interpreter = PythonInterpreter(new SparkConf(), PySpark())
+}
+
+class Python3SessionSpec extends PythonSessionSpec {
+
+ override protected def withFixture(test: NoArgTest): Outcome = {
+ assume(!sys.props.getOrElse("skipPySpark3Tests", "false").toBoolean, "Skipping PySpark3 tests.")
+ test()
+ }
+
+ override def createInterpreter(): Interpreter = PythonInterpreter(new SparkConf(), PySpark3())
+
+ it should "check python version is 3.x" in withSession { session =>
+ val statement = execute(session)(
+ """import sys
+ |sys.version >= '3'
+ """.stripMargin)
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "True"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/org/apache/livy/repl/ReplDriverSuite.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/livy/repl/ReplDriverSuite.scala b/repl/src/test/scala/org/apache/livy/repl/ReplDriverSuite.scala
new file mode 100644
index 0000000..6537f0c
--- /dev/null
+++ b/repl/src/test/scala/org/apache/livy/repl/ReplDriverSuite.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import java.net.URI
+import java.util.concurrent.TimeUnit
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.apache.spark.launcher.SparkLauncher
+import org.json4s._
+import org.json4s.jackson.JsonMethods._
+import org.scalatest.FunSuite
+import org.scalatest.concurrent.Eventually._
+
+import org.apache.livy._
+import org.apache.livy.rsc.{PingJob, RSCClient, RSCConf}
+import org.apache.livy.sessions.Spark
+
+class ReplDriverSuite extends FunSuite with LivyBaseUnitTestSuite {
+
+ private implicit val formats = DefaultFormats
+
+ test("start a repl session using the rsc") {
+ val client = new LivyClientBuilder()
+ .setConf(SparkLauncher.DRIVER_MEMORY, "512m")
+ .setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, sys.props("java.class.path"))
+ .setConf(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH, sys.props("java.class.path"))
+ .setConf(RSCConf.Entry.LIVY_JARS.key(), "")
+ .setURI(new URI("rsc:/"))
+ .setConf(RSCConf.Entry.DRIVER_CLASS.key(), classOf[ReplDriver].getName())
+ .setConf(RSCConf.Entry.SESSION_KIND.key(), Spark().toString)
+ .build()
+ .asInstanceOf[RSCClient]
+
+ try {
+ // This is sort of what InteractiveSession.scala does to detect an idle session.
+ client.submit(new PingJob()).get(60, TimeUnit.SECONDS)
+
+ val statementId = client.submitReplCode("1 + 1").get
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ val rawResult =
+ client.getReplJobResults(statementId, 1).get(10, TimeUnit.SECONDS).statements(0)
+ val result = rawResult.output
+ assert((parse(result) \ Session.STATUS).extract[String] === Session.OK)
+ }
+ } finally {
+ client.stop(true)
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/org/apache/livy/repl/ScalaInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/livy/repl/ScalaInterpreterSpec.scala b/repl/src/test/scala/org/apache/livy/repl/ScalaInterpreterSpec.scala
new file mode 100644
index 0000000..3e9ee82
--- /dev/null
+++ b/repl/src/test/scala/org/apache/livy/repl/ScalaInterpreterSpec.scala
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import org.apache.spark.SparkConf
+import org.json4s.{DefaultFormats, JValue}
+import org.json4s.JsonDSL._
+
+import org.apache.livy.rsc.RSCConf
+
+class ScalaInterpreterSpec extends BaseInterpreterSpec {
+
+ implicit val formats = DefaultFormats
+
+ override def createInterpreter(): Interpreter =
+ new SparkInterpreter(new SparkConf())
+
+ it should "execute `1 + 2` == 3" in withInterpreter { interpreter =>
+ val response = interpreter.execute("1 + 2")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "res0: Int = 3"
+ ))
+ }
+
+ it should "execute multiple statements" in withInterpreter { interpreter =>
+ var response = interpreter.execute("val x = 1")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "x: Int = 1"
+ ))
+
+ response = interpreter.execute("val y = 2")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "y: Int = 2"
+ ))
+
+ response = interpreter.execute("x + y")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "res0: Int = 3"
+ ))
+ }
+
+ it should "execute multiple statements in one block" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """
+ |val x = 1
+ |
+ |val y = 2
+ |
+ |x + y
+ """.stripMargin)
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "res2: Int = 3"
+ ))
+ }
+
+ it should "do table magic" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """val x = List(List(1, "a"), List(3, "b"))
+ |%table x
+ """.stripMargin)
+
+ response should equal(Interpreter.ExecuteSuccess(
+ APPLICATION_LIVY_TABLE_JSON -> (
+ ("headers" -> List(
+ ("type" -> "BIGINT_TYPE") ~ ("name" -> "0"),
+ ("type" -> "STRING_TYPE") ~ ("name" -> "1")
+ )) ~
+ ("data" -> List(
+ List[JValue](1, "a"),
+ List[JValue](3, "b")
+ ))
+ )
+ ))
+ }
+
+ it should "allow magic inside statements" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """val x = List(List(1, "a"), List(3, "b"))
+ |%table x
+ |1 + 2
+ """.stripMargin)
+
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "res0: Int = 3"
+ ))
+ }
+
+ it should "capture stdout" in withInterpreter { interpreter =>
+ val response = interpreter.execute("println(\"Hello World\")")
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "Hello World"
+ ))
+ }
+
+ it should "report an error if accessing an unknown variable" in withInterpreter { interpreter =>
+ interpreter.execute("x") match {
+ case Interpreter.ExecuteError(ename, evalue, _) =>
+ ename should equal ("Error")
+ evalue should include ("error: not found: value x")
+
+ case other =>
+ fail(s"Expected error, got $other.")
+ }
+ }
+
+ it should "execute spark commands" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """sc.parallelize(0 to 1).map { i => i+1 }.collect""".stripMargin)
+
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "res0: Array[Int] = Array(1, 2)"
+ ))
+ }
+
+ it should "handle statements ending with comments" in withInterpreter { interpreter =>
+ // Test statements with only comments
+ var response = interpreter.execute("""// comment""")
+ response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> ""))
+
+ response = interpreter.execute(
+ """/*
+ |comment
+ |*/
+ """.stripMargin)
+ response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> ""))
+
+ // Test statements ending with comments
+ response = interpreter.execute(
+ """val r = 1
+ |// comment
+ """.stripMargin)
+ response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> "r: Int = 1"))
+
+ response = interpreter.execute(
+ """val r = 1
+ |/*
+ |comment
+ |comment
+ |*/
+ """.stripMargin)
+ response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> "r: Int = 1"))
+
+ // Test statements ending with a mix of single line and multi-line comments
+ response = interpreter.execute(
+ """val r = 1
+ |// comment
+ |/*
+ |comment
+ |comment
+ |*/
+ |// comment
+ """.stripMargin)
+ response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> "r: Int = 1"))
+
+ response = interpreter.execute(
+ """val r = 1
+ |/*
+ |comment
+ |// comment
+ |comment
+ |*/
+ """.stripMargin)
+ response should equal(Interpreter.ExecuteSuccess(TEXT_PLAIN -> "r: Int = 1"))
+
+ // Make sure incomplete statement is still returned as incomplete statement.
+ response = interpreter.execute("sc.")
+ response should equal(Interpreter.ExecuteIncomplete())
+
+ // Make sure incomplete statement is still returned as incomplete statement.
+ response = interpreter.execute(
+ """sc.
+ |// comment
+ """.stripMargin)
+ response should equal(Interpreter.ExecuteIncomplete())
+
+ // Make sure our handling doesn't mess up a string with value like comments.
+ val tripleQuotes = "\"\"\""
+ val stringWithComment = s"/*\ncomment\n*/\n//comment"
+ response = interpreter.execute(s"val r = $tripleQuotes$stringWithComment$tripleQuotes")
+
+ try {
+ response should equal(
+ Interpreter.ExecuteSuccess(TEXT_PLAIN -> s"r: String = \n$stringWithComment"))
+ } catch {
+ case _: Exception =>
+ response should equal(
+ // Scala 2.11 doesn't have a " " after "="
+ Interpreter.ExecuteSuccess(TEXT_PLAIN -> s"r: String =\n$stringWithComment"))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/org/apache/livy/repl/SessionSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/livy/repl/SessionSpec.scala b/repl/src/test/scala/org/apache/livy/repl/SessionSpec.scala
new file mode 100644
index 0000000..090b7cb
--- /dev/null
+++ b/repl/src/test/scala/org/apache/livy/repl/SessionSpec.scala
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import java.util.Properties
+import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit}
+
+import org.mockito.Mockito.when
+import org.mockito.invocation.InvocationOnMock
+import org.mockito.stubbing.Answer
+import org.scalatest.FunSpec
+import org.scalatest.Matchers._
+import org.scalatest.concurrent.Eventually
+import org.scalatest.mock.MockitoSugar.mock
+import org.scalatest.time._
+
+import org.apache.livy.LivyBaseUnitTestSuite
+import org.apache.livy.repl.Interpreter.ExecuteResponse
+import org.apache.livy.rsc.RSCConf
+
+class SessionSpec extends FunSpec with Eventually with LivyBaseUnitTestSuite {
+ override implicit val patienceConfig =
+ PatienceConfig(timeout = scaled(Span(10, Seconds)), interval = scaled(Span(100, Millis)))
+
+ private val rscConf = new RSCConf(new Properties())
+
+ describe("Session") {
+ it("should call state changed callbacks in happy path") {
+ val expectedStateTransitions =
+ Array("not_started", "starting", "idle", "busy", "idle", "busy", "idle")
+ val actualStateTransitions = new ConcurrentLinkedQueue[String]()
+
+ val interpreter = mock[Interpreter]
+ when(interpreter.kind).thenAnswer(new Answer[String] {
+ override def answer(invocationOnMock: InvocationOnMock): String = "spark"
+ })
+
+ val session =
+ new Session(rscConf, interpreter, { s => actualStateTransitions.add(s.toString) })
+
+ session.start()
+
+ session.execute("")
+
+ eventually {
+ actualStateTransitions.toArray shouldBe expectedStateTransitions
+ }
+ }
+
+ it("should not transit to idle if there're any pending statements.") {
+ val expectedStateTransitions =
+ Array("not_started", "busy", "busy", "busy", "idle", "busy", "idle")
+ val actualStateTransitions = new ConcurrentLinkedQueue[String]()
+
+ val interpreter = mock[Interpreter]
+ when(interpreter.kind).thenAnswer(new Answer[String] {
+ override def answer(invocationOnMock: InvocationOnMock): String = "spark"
+ })
+
+ val blockFirstExecuteCall = new CountDownLatch(1)
+ when(interpreter.execute("")).thenAnswer(new Answer[Interpreter.ExecuteResponse] {
+ override def answer(invocation: InvocationOnMock): ExecuteResponse = {
+ blockFirstExecuteCall.await(10, TimeUnit.SECONDS)
+ null
+ }
+ })
+ val session =
+ new Session(rscConf, interpreter, { s => actualStateTransitions.add(s.toString) })
+
+ for (_ <- 1 to 2) {
+ session.execute("")
+ }
+
+ blockFirstExecuteCall.countDown()
+ eventually {
+ actualStateTransitions.toArray shouldBe expectedStateTransitions
+ }
+ }
+
+ it("should remove old statements when reaching threshold") {
+ val interpreter = mock[Interpreter]
+ when(interpreter.kind).thenAnswer(new Answer[String] {
+ override def answer(invocationOnMock: InvocationOnMock): String = "spark"
+ })
+
+ rscConf.set(RSCConf.Entry.RETAINED_STATEMENT_NUMBER, 2)
+ val session = new Session(rscConf, interpreter)
+ session.start()
+
+ session.statements.size should be (0)
+ session.execute("")
+ session.statements.size should be (1)
+ session.statements.map(_._1).toSet should be (Set(0))
+ session.execute("")
+ session.statements.size should be (2)
+ session.statements.map(_._1).toSet should be (Set(0, 1))
+ session.execute("")
+ eventually {
+ session.statements.size should be (2)
+ session.statements.map(_._1).toSet should be (Set(1, 2))
+ }
+
+ // Continue submitting statements, total statements in memory should be 2.
+ session.execute("")
+ eventually {
+ session.statements.size should be (2)
+ session.statements.map(_._1).toSet should be (Set(2, 3))
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/org/apache/livy/repl/SparkRInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/livy/repl/SparkRInterpreterSpec.scala b/repl/src/test/scala/org/apache/livy/repl/SparkRInterpreterSpec.scala
new file mode 100644
index 0000000..374afbc
--- /dev/null
+++ b/repl/src/test/scala/org/apache/livy/repl/SparkRInterpreterSpec.scala
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import org.apache.spark.SparkConf
+import org.json4s.{DefaultFormats, JValue}
+import org.json4s.JsonDSL._
+import org.scalatest._
+
+import org.apache.livy.rsc.RSCConf
+
+class SparkRInterpreterSpec extends BaseInterpreterSpec {
+
+ implicit val formats = DefaultFormats
+
+ override protected def withFixture(test: NoArgTest): Outcome = {
+ assume(!sys.props.getOrElse("skipRTests", "false").toBoolean, "Skipping R tests.")
+ super.withFixture(test)
+ }
+
+ override def createInterpreter(): Interpreter = SparkRInterpreter(new SparkConf())
+
+ it should "execute `1 + 2` == 3" in withInterpreter { interpreter =>
+ val response = interpreter.execute("1 + 2")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "[1] 3"
+ ))
+ }
+
+ it should "execute multiple statements" in withInterpreter { interpreter =>
+ var response = interpreter.execute("x = 1")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> ""
+ ))
+
+ response = interpreter.execute("y = 2")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> ""
+ ))
+
+ response = interpreter.execute("x + y")
+ response should equal (Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "[1] 3"
+ ))
+ }
+
+ it should "execute multiple statements in one block" in withInterpreter { interpreter =>
+ val response = interpreter.execute(
+ """
+ |x = 1
+ |
+ |y = 2
+ |
+ |x + y
+ """.stripMargin)
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "[1] 3"
+ ))
+ }
+
+ it should "capture stdout" in withInterpreter { interpreter =>
+ val response = interpreter.execute("cat(3)")
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "3"
+ ))
+ }
+
+ it should "report an error if accessing an unknown variable" in withInterpreter { interpreter =>
+ val response = interpreter.execute("x")
+ assert(response.isInstanceOf[Interpreter.ExecuteError])
+ val errorResponse = response.asInstanceOf[Interpreter.ExecuteError]
+ errorResponse.ename should be ("Error")
+ assert(errorResponse.evalue.contains("object 'x' not found"))
+ }
+
+
+ it should "not hang when executing incomplete statements" in withInterpreter { interpreter =>
+ val response = interpreter.execute("x[")
+ response should equal(Interpreter.ExecuteError(
+ "Error",
+ """[1] "Error in parse(text = \"x[\"): <text>:2:0: unexpected end of input\n1: x[\n ^""""
+ ))
+ }
+
+ it should "escape the statement" in withInterpreter { interpreter =>
+ val response = interpreter.execute("print(\"a\")")
+ response should equal(Interpreter.ExecuteSuccess(
+ TEXT_PLAIN -> "[1] \"a\""
+ ))
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/org/apache/livy/repl/SparkRSessionSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/livy/repl/SparkRSessionSpec.scala b/repl/src/test/scala/org/apache/livy/repl/SparkRSessionSpec.scala
new file mode 100644
index 0000000..42ed60a
--- /dev/null
+++ b/repl/src/test/scala/org/apache/livy/repl/SparkRSessionSpec.scala
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import org.apache.spark.SparkConf
+import org.json4s.Extraction
+import org.json4s.jackson.JsonMethods.parse
+
+import org.apache.livy.rsc.RSCConf
+
+class SparkRSessionSpec extends BaseSessionSpec {
+
+ override protected def withFixture(test: NoArgTest) = {
+ assume(!sys.props.getOrElse("skipRTests", "false").toBoolean, "Skipping R tests.")
+ super.withFixture(test)
+ }
+
+ override def createInterpreter(): Interpreter = SparkRInterpreter(new SparkConf())
+
+ it should "execute `1 + 2` == 3" in withSession { session =>
+ val statement = execute(session)("1 + 2")
+ statement.id should equal(0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "[1] 3"
+ )
+ ))
+
+ result should equal(expectedResult)
+ }
+
+ it should "execute `x = 1`, then `y = 2`, then `x + y`" in withSession { session =>
+ val executeWithSession = execute(session)(_)
+ var statement = executeWithSession("x = 1")
+ statement.id should equal (0)
+
+ var result = parse(statement.output)
+ var expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> ""
+ )
+ ))
+
+ result should equal (expectedResult)
+
+ statement = executeWithSession("y = 2")
+ statement.id should equal (1)
+
+ result = parse(statement.output)
+ expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 1,
+ "data" -> Map(
+ "text/plain" -> ""
+ )
+ ))
+
+ result should equal (expectedResult)
+
+ statement = executeWithSession("x + y")
+ statement.id should equal (2)
+
+ result = parse(statement.output)
+ expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 2,
+ "data" -> Map(
+ "text/plain" -> "[1] 3"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "capture stdout from print" in withSession { session =>
+ val statement = execute(session)("""print('Hello World')""")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "[1] \"Hello World\""
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "capture stdout from cat" in withSession { session =>
+ val statement = execute(session)("""cat(3)""")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "3"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "report an error if accessing an unknown variable" in withSession { session =>
+ val statement = execute(session)("""x""")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ (result \ "status").extract[String] should be ("error")
+ (result \ "execution_count").extract[Int] should be (0)
+ (result \ "ename").extract[String] should be ("Error")
+ assert((result \ "evalue").extract[String].contains("object 'x' not found"))
+ (result \ "traceback").extract[List[String]] should be (List())
+ }
+
+}
[07/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/utils/LivySparkUtils.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/utils/LivySparkUtils.scala b/server/src/main/scala/com/cloudera/livy/utils/LivySparkUtils.scala
deleted file mode 100644
index ab1e7ff..0000000
--- a/server/src/main/scala/com/cloudera/livy/utils/LivySparkUtils.scala
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.utils
-
-import java.io.{File, IOException}
-
-import scala.collection.SortedMap
-import scala.math.Ordering.Implicits._
-
-import com.cloudera.livy.{LivyConf, Logging}
-import com.cloudera.livy.LivyConf.LIVY_SPARK_SCALA_VERSION
-import com.cloudera.livy.util.LineBufferedProcess
-
-object LivySparkUtils extends Logging {
-
- // For each Spark version we supported, we need to add this mapping relation in case Scala
- // version cannot be detected from "spark-submit --version".
- private val _defaultSparkScalaVersion = SortedMap(
- // Spark 2.1 + Scala 2.11
- (2, 1) -> "2.11",
- // Spark 2.0 + Scala 2.11
- (2, 0) -> "2.11",
- // Spark 1.6 + Scala 2.10
- (1, 6) -> "2.10"
- )
-
- // Supported Spark version
- private val MIN_VERSION = (1, 6)
- private val MAX_VERSION = (2, 2)
-
- private val sparkVersionRegex = """version (.*)""".r.unanchored
- private val scalaVersionRegex = """Scala version (.*), Java""".r.unanchored
-
- /**
- * Test that Spark home is configured and configured Spark home is a directory.
- */
- def testSparkHome(livyConf: LivyConf): Unit = {
- val sparkHome = livyConf.sparkHome().getOrElse {
- throw new IllegalArgumentException("Livy requires the SPARK_HOME environment variable")
- }
-
- require(new File(sparkHome).isDirectory(), "SPARK_HOME path does not exist")
- }
-
- /**
- * Test that the configured `spark-submit` executable exists.
- *
- * @param livyConf
- */
- def testSparkSubmit(livyConf: LivyConf): Unit = {
- try {
- testSparkVersion(sparkSubmitVersion(livyConf)._1)
- } catch {
- case e: IOException =>
- throw new IOException("Failed to run spark-submit executable", e)
- }
- }
-
- /**
- * Throw an exception if Spark version is not supported.
- * @param version Spark version
- */
- def testSparkVersion(version: String): Unit = {
- val v = formatSparkVersion(version)
- require(v >= MIN_VERSION, s"Unsupported Spark version $v")
- if (v >= MAX_VERSION) {
- warn(s"Current Spark $v is not verified in Livy, please use it carefully")
- }
- }
-
- /**
- * Call `spark-submit --version` and parse its output for Spark and Scala version.
- *
- * @param livyConf
- * @return Tuple with Spark and Scala version
- */
- def sparkSubmitVersion(livyConf: LivyConf): (String, Option[String]) = {
- val sparkSubmit = livyConf.sparkSubmit()
- val pb = new ProcessBuilder(sparkSubmit, "--version")
- pb.redirectErrorStream(true)
- pb.redirectInput(ProcessBuilder.Redirect.PIPE)
-
- if (LivyConf.TEST_MODE) {
- pb.environment().put("LIVY_TEST_CLASSPATH", sys.props("java.class.path"))
- }
-
- val process = new LineBufferedProcess(pb.start())
- val exitCode = process.waitFor()
- val output = process.inputIterator.mkString("\n")
-
- var sparkVersion = ""
- output match {
- case sparkVersionRegex(version) => sparkVersion = version
- case _ =>
- throw new IOException(f"Unable to determine spark-submit version [$exitCode]:\n$output")
- }
-
- val scalaVersion = output match {
- case scalaVersionRegex(version) if version.nonEmpty => Some(formatScalaVersion(version))
- case _ => None
- }
-
- (sparkVersion, scalaVersion)
- }
-
- def sparkScalaVersion(
- formattedSparkVersion: (Int, Int),
- scalaVersionFromSparkSubmit: Option[String],
- livyConf: LivyConf): String = {
- val scalaVersionInLivyConf = Option(livyConf.get(LIVY_SPARK_SCALA_VERSION))
- .filter(_.nonEmpty)
- .map(formatScalaVersion)
-
- for (vSparkSubmit <- scalaVersionFromSparkSubmit; vLivyConf <- scalaVersionInLivyConf) {
- require(vSparkSubmit == vLivyConf,
- s"Scala version detected from spark-submit ($vSparkSubmit) does not match " +
- s"Scala version configured in livy.conf ($vLivyConf)")
- }
-
- scalaVersionInLivyConf
- .orElse(scalaVersionFromSparkSubmit)
- .getOrElse(defaultSparkScalaVersion(formattedSparkVersion))
- }
-
- /**
- * Return formatted Spark version.
- *
- * @param version Spark version
- * @return Two element tuple, one is major version and the other is minor version
- */
- def formatSparkVersion(version: String): (Int, Int) = {
- val versionPattern = """^(\d+)\.(\d+)(\..*)?$""".r
- versionPattern.findFirstMatchIn(version) match {
- case Some(m) =>
- (m.group(1).toInt, m.group(2).toInt)
- case None =>
- throw new IllegalArgumentException(s"Fail to parse Spark version from $version")
- }
- }
-
- /**
- * Return Scala binary version.
- * It strips the patch version if specified.
- * Throws if it cannot parse the version.
- *
- * @param scalaVersion Scala binary version String
- * @return Scala binary version
- */
- def formatScalaVersion(scalaVersion: String): String = {
- val versionPattern = """(\d)+\.(\d+)+.*""".r
- scalaVersion match {
- case versionPattern(major, minor) => s"$major.$minor"
- case _ => throw new IllegalArgumentException(s"Unrecognized Scala version: $scalaVersion")
- }
- }
-
- /**
- * Return the default Scala version of a Spark version.
- *
- * @param sparkVersion formatted Spark version.
- * @return Scala binary version
- */
- private[utils] def defaultSparkScalaVersion(sparkVersion: (Int, Int)): String = {
- _defaultSparkScalaVersion.get(sparkVersion)
- .orElse {
- if (sparkVersion < _defaultSparkScalaVersion.head._1) {
- throw new IllegalArgumentException(s"Spark version $sparkVersion is less than the " +
- s"minimum version ${_defaultSparkScalaVersion.head._1} supported by Livy")
- } else if (sparkVersion > _defaultSparkScalaVersion.last._1) {
- val (spark, scala) = _defaultSparkScalaVersion.last
- warn(s"Spark version $sparkVersion is greater then the maximum version " +
- s"$spark supported by Livy, will choose Scala version $scala instead, " +
- s"please specify manually if it is the expected Scala version you want")
- Some(scala)
- } else {
- None
- }
- }
- .getOrElse(
- throw new IllegalArgumentException(s"Fail to get Scala version from Spark $sparkVersion"))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/utils/SparkApp.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/utils/SparkApp.scala b/server/src/main/scala/com/cloudera/livy/utils/SparkApp.scala
deleted file mode 100644
index 1644518..0000000
--- a/server/src/main/scala/com/cloudera/livy/utils/SparkApp.scala
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.utils
-
-import scala.collection.JavaConverters._
-
-import com.cloudera.livy.LivyConf
-import com.cloudera.livy.util.LineBufferedProcess
-
-object AppInfo {
- val DRIVER_LOG_URL_NAME = "driverLogUrl"
- val SPARK_UI_URL_NAME = "sparkUiUrl"
-}
-
-case class AppInfo(var driverLogUrl: Option[String] = None, var sparkUiUrl: Option[String] = None) {
- import AppInfo._
- def asJavaMap: java.util.Map[String, String] =
- Map(DRIVER_LOG_URL_NAME -> driverLogUrl.orNull, SPARK_UI_URL_NAME -> sparkUiUrl.orNull).asJava
-}
-
-trait SparkAppListener {
- /** Fired when appId is known, even during recovery. */
- def appIdKnown(appId: String): Unit = {}
-
- /** Fired when the app state in the cluster changes. */
- def stateChanged(oldState: SparkApp.State, newState: SparkApp.State): Unit = {}
-
- /** Fired when the app info is changed. */
- def infoChanged(appInfo: AppInfo): Unit = {}
-}
-
-/**
- * Provide factory methods for SparkApp.
- */
-object SparkApp {
- private val SPARK_YARN_TAG_KEY = "spark.yarn.tags"
-
- object State extends Enumeration {
- val STARTING, RUNNING, FINISHED, FAILED, KILLED = Value
- }
- type State = State.Value
-
- /**
- * Return cluster manager dependent SparkConf.
- *
- * @param uniqueAppTag A tag that can uniquely identify the application.
- * @param livyConf
- * @param sparkConf
- */
- def prepareSparkConf(
- uniqueAppTag: String,
- livyConf: LivyConf,
- sparkConf: Map[String, String]): Map[String, String] = {
- if (livyConf.isRunningOnYarn()) {
- val userYarnTags = sparkConf.get(SPARK_YARN_TAG_KEY).map("," + _).getOrElse("")
- val mergedYarnTags = uniqueAppTag + userYarnTags
- sparkConf ++ Map(
- SPARK_YARN_TAG_KEY -> mergedYarnTags,
- "spark.yarn.submit.waitAppCompletion" -> "false")
- } else {
- sparkConf
- }
- }
-
- /**
- * Return a SparkApp object to control the underlying Spark application via YARN or spark-submit.
- *
- * @param uniqueAppTag A tag that can uniquely identify the application.
- */
- def create(
- uniqueAppTag: String,
- appId: Option[String],
- process: Option[LineBufferedProcess],
- livyConf: LivyConf,
- listener: Option[SparkAppListener]): SparkApp = {
- if (livyConf.isRunningOnYarn()) {
- new SparkYarnApp(uniqueAppTag, appId, process, listener, livyConf)
- } else {
- require(process.isDefined, "process must not be None when Livy master is not YARN.")
- new SparkProcApp(process.get, listener)
- }
- }
-}
-
-/**
- * Encapsulate a Spark application.
- */
-abstract class SparkApp {
- def kill(): Unit
- def log(): IndexedSeq[String]
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/utils/SparkProcApp.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/utils/SparkProcApp.scala b/server/src/main/scala/com/cloudera/livy/utils/SparkProcApp.scala
deleted file mode 100644
index 4fde7b6..0000000
--- a/server/src/main/scala/com/cloudera/livy/utils/SparkProcApp.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.utils
-
-import com.cloudera.livy.{Logging, Utils}
-import com.cloudera.livy.util.LineBufferedProcess
-
-/**
- * Provide a class to control a Spark application using spark-submit.
- *
- * @param process The spark-submit process launched the Spark application.
- */
-class SparkProcApp (
- process: LineBufferedProcess,
- listener: Option[SparkAppListener])
- extends SparkApp with Logging {
-
- private var state = SparkApp.State.STARTING
-
- override def kill(): Unit = {
- if (process.isAlive) {
- process.destroy()
- waitThread.join()
- }
- }
-
- override def log(): IndexedSeq[String] = process.inputLines
-
- private def changeState(newState: SparkApp.State.Value) = {
- if (state != newState) {
- listener.foreach(_.stateChanged(state, newState))
- state = newState
- }
- }
-
- private val waitThread = Utils.startDaemonThread(s"SparProcApp_$this") {
- changeState(SparkApp.State.RUNNING)
- process.waitFor() match {
- case 0 => changeState(SparkApp.State.FINISHED)
- case exitCode =>
- changeState(SparkApp.State.FAILED)
- error(s"spark-submit exited with code $exitCode")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/utils/SparkProcessBuilder.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/utils/SparkProcessBuilder.scala b/server/src/main/scala/com/cloudera/livy/utils/SparkProcessBuilder.scala
deleted file mode 100644
index 445ef99..0000000
--- a/server/src/main/scala/com/cloudera/livy/utils/SparkProcessBuilder.scala
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.utils
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-
-import com.cloudera.livy.{LivyConf, Logging}
-import com.cloudera.livy.util.LineBufferedProcess
-
-class SparkProcessBuilder(livyConf: LivyConf) extends Logging {
-
- private[this] var _executable: String = livyConf.sparkSubmit()
- private[this] var _master: Option[String] = None
- private[this] var _deployMode: Option[String] = None
- private[this] var _className: Option[String] = None
- private[this] var _name: Option[String] = None
- private[this] val _conf = mutable.HashMap[String, String]()
- private[this] var _driverClassPath: ArrayBuffer[String] = ArrayBuffer()
- private[this] var _proxyUser: Option[String] = None
- private[this] var _queue: Option[String] = None
- private[this] var _env: ArrayBuffer[(String, String)] = ArrayBuffer()
- private[this] var _redirectOutput: Option[ProcessBuilder.Redirect] = None
- private[this] var _redirectError: Option[ProcessBuilder.Redirect] = None
- private[this] var _redirectErrorStream: Option[Boolean] = None
-
- def executable(executable: String): SparkProcessBuilder = {
- _executable = executable
- this
- }
-
- def master(masterUrl: String): SparkProcessBuilder = {
- _master = Some(masterUrl)
- this
- }
-
- def deployMode(deployMode: String): SparkProcessBuilder = {
- _deployMode = Some(deployMode)
- this
- }
-
- def className(className: String): SparkProcessBuilder = {
- _className = Some(className)
- this
- }
-
- def name(name: String): SparkProcessBuilder = {
- _name = Some(name)
- this
- }
-
- def conf(key: String): Option[String] = {
- _conf.get(key)
- }
-
- def conf(key: String, value: String, admin: Boolean = false): SparkProcessBuilder = {
- this._conf(key) = value
- this
- }
-
- def conf(conf: Traversable[(String, String)]): SparkProcessBuilder = {
- conf.foreach { case (key, value) => this.conf(key, value) }
- this
- }
-
- def driverJavaOptions(driverJavaOptions: String): SparkProcessBuilder = {
- conf("spark.driver.extraJavaOptions", driverJavaOptions)
- }
-
- def driverClassPath(classPath: String): SparkProcessBuilder = {
- _driverClassPath += classPath
- this
- }
-
- def driverClassPaths(classPaths: Traversable[String]): SparkProcessBuilder = {
- _driverClassPath ++= classPaths
- this
- }
-
- def driverCores(driverCores: Int): SparkProcessBuilder = {
- this.driverCores(driverCores.toString)
- }
-
- def driverMemory(driverMemory: String): SparkProcessBuilder = {
- conf("spark.driver.memory", driverMemory)
- }
-
- def driverCores(driverCores: String): SparkProcessBuilder = {
- conf("spark.driver.cores", driverCores)
- }
-
- def executorCores(executorCores: Int): SparkProcessBuilder = {
- this.executorCores(executorCores.toString)
- }
-
- def executorCores(executorCores: String): SparkProcessBuilder = {
- conf("spark.executor.cores", executorCores)
- }
-
- def executorMemory(executorMemory: String): SparkProcessBuilder = {
- conf("spark.executor.memory", executorMemory)
- }
-
- def numExecutors(numExecutors: Int): SparkProcessBuilder = {
- this.numExecutors(numExecutors.toString)
- }
-
- def numExecutors(numExecutors: String): SparkProcessBuilder = {
- this.conf("spark.executor.instances", numExecutors)
- }
-
- def proxyUser(proxyUser: String): SparkProcessBuilder = {
- _proxyUser = Some(proxyUser)
- this
- }
-
- def queue(queue: String): SparkProcessBuilder = {
- _queue = Some(queue)
- this
- }
-
- def env(key: String, value: String): SparkProcessBuilder = {
- _env += ((key, value))
- this
- }
-
- def redirectOutput(redirect: ProcessBuilder.Redirect): SparkProcessBuilder = {
- _redirectOutput = Some(redirect)
- this
- }
-
- def redirectError(redirect: ProcessBuilder.Redirect): SparkProcessBuilder = {
- _redirectError = Some(redirect)
- this
- }
-
- def redirectErrorStream(redirect: Boolean): SparkProcessBuilder = {
- _redirectErrorStream = Some(redirect)
- this
- }
-
- def start(file: Option[String], args: Traversable[String]): LineBufferedProcess = {
- var arguments = ArrayBuffer(_executable)
-
- def addOpt(option: String, value: Option[String]): Unit = {
- value.foreach { v =>
- arguments += option
- arguments += v
- }
- }
-
- def addList(option: String, values: Traversable[String]): Unit = {
- if (values.nonEmpty) {
- arguments += option
- arguments += values.mkString(",")
- }
- }
-
- addOpt("--master", _master)
- addOpt("--deploy-mode", _deployMode)
- addOpt("--name", _name)
- addOpt("--class", _className)
- _conf.foreach { case (key, value) =>
- if (key == "spark.submit.pyFiles") {
- arguments += "--py-files"
- arguments += f"$value"
- } else {
- arguments += "--conf"
- arguments += f"$key=$value"
- }
- }
- addList("--driver-class-path", _driverClassPath)
-
- if (livyConf.getBoolean(LivyConf.IMPERSONATION_ENABLED)) {
- addOpt("--proxy-user", _proxyUser)
- }
-
- addOpt("--queue", _queue)
-
- arguments += file.getOrElse("spark-internal")
- arguments ++= args
-
- val argsString = arguments
- .map("'" + _.replace("'", "\\'") + "'")
- .mkString(" ")
-
- info(s"Running $argsString")
-
- val pb = new ProcessBuilder(arguments.asJava)
- val env = pb.environment()
-
- for ((key, value) <- _env) {
- env.put(key, value)
- }
-
- _redirectOutput.foreach(pb.redirectOutput)
- _redirectError.foreach(pb.redirectError)
- _redirectErrorStream.foreach(pb.redirectErrorStream)
-
- new LineBufferedProcess(pb.start())
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/utils/SparkYarnApp.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/utils/SparkYarnApp.scala b/server/src/main/scala/com/cloudera/livy/utils/SparkYarnApp.scala
deleted file mode 100644
index 05a3d53..0000000
--- a/server/src/main/scala/com/cloudera/livy/utils/SparkYarnApp.scala
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.utils
-
-import java.util.concurrent.TimeoutException
-
-import scala.annotation.tailrec
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-import scala.concurrent._
-import scala.concurrent.duration._
-import scala.language.postfixOps
-import scala.util.Try
-
-import org.apache.hadoop.yarn.api.records.{ApplicationId, ApplicationReport, FinalApplicationStatus, YarnApplicationState}
-import org.apache.hadoop.yarn.client.api.YarnClient
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException
-import org.apache.hadoop.yarn.util.ConverterUtils
-
-import com.cloudera.livy.{LivyConf, Logging, Utils}
-import com.cloudera.livy.util.LineBufferedProcess
-
-object SparkYarnApp extends Logging {
-
- def init(livyConf: LivyConf): Unit = {
- sessionLeakageCheckInterval = livyConf.getTimeAsMs(LivyConf.YARN_APP_LEAKAGE_CHECK_INTERVAL)
- sessionLeakageCheckTimeout = livyConf.getTimeAsMs(LivyConf.YARN_APP_LEAKAGE_CHECK_TIMEOUT)
- leakedAppsGCThread.setDaemon(true)
- leakedAppsGCThread.setName("LeakedAppsGCThread")
- leakedAppsGCThread.start()
- }
-
- // YarnClient is thread safe. Create once, share it across threads.
- lazy val yarnClient = {
- val c = YarnClient.createYarnClient()
- c.init(new YarnConfiguration())
- c.start()
- c
- }
-
- private def getYarnTagToAppIdTimeout(livyConf: LivyConf): FiniteDuration =
- livyConf.getTimeAsMs(LivyConf.YARN_APP_LOOKUP_TIMEOUT) milliseconds
-
- private def getYarnPollInterval(livyConf: LivyConf): FiniteDuration =
- livyConf.getTimeAsMs(LivyConf.YARN_POLL_INTERVAL) milliseconds
-
- private val appType = Set("SPARK").asJava
-
- private val leakedAppTags = new java.util.concurrent.ConcurrentHashMap[String, Long]()
-
- private var sessionLeakageCheckTimeout: Long = _
-
- private var sessionLeakageCheckInterval: Long = _
-
- private val leakedAppsGCThread = new Thread() {
- override def run(): Unit = {
- while (true) {
- if (!leakedAppTags.isEmpty) {
- // kill the app if found it and remove it if exceeding a threashold
- val iter = leakedAppTags.entrySet().iterator()
- var isRemoved = false
- val now = System.currentTimeMillis()
- val apps = yarnClient.getApplications(appType).asScala
- while(iter.hasNext) {
- val entry = iter.next()
- apps.find(_.getApplicationTags.contains(entry.getKey))
- .foreach({ e =>
- info(s"Kill leaked app ${e.getApplicationId}")
- yarnClient.killApplication(e.getApplicationId)
- iter.remove()
- isRemoved = true
- })
- if (!isRemoved) {
- if ((entry.getValue - now) > sessionLeakageCheckTimeout) {
- iter.remove()
- info(s"Remove leaked yarn app tag ${entry.getKey}")
- }
- }
- }
- }
- Thread.sleep(sessionLeakageCheckInterval)
- }
- }
- }
-
-
-}
-
-/**
- * Provide a class to control a Spark application using YARN API.
- *
- * @param appTag An app tag that can unique identify the YARN app.
- * @param appIdOption The appId of the YARN app. If this's None, SparkYarnApp will find it
- * using appTag.
- * @param process The spark-submit process launched the YARN application. This is optional.
- * If it's provided, SparkYarnApp.log() will include its log.
- * @param listener Optional listener for notification of appId discovery and app state changes.
- */
-class SparkYarnApp private[utils] (
- appTag: String,
- appIdOption: Option[String],
- process: Option[LineBufferedProcess],
- listener: Option[SparkAppListener],
- livyConf: LivyConf,
- yarnClient: => YarnClient = SparkYarnApp.yarnClient) // For unit test.
- extends SparkApp
- with Logging {
- import SparkYarnApp._
-
- private val appIdPromise: Promise[ApplicationId] = Promise()
- private[utils] var state: SparkApp.State = SparkApp.State.STARTING
- private var yarnDiagnostics: IndexedSeq[String] = IndexedSeq.empty[String]
-
- override def log(): IndexedSeq[String] =
- ("stdout: " +: process.map(_.inputLines).getOrElse(ArrayBuffer.empty[String])) ++
- ("\nstderr: " +: process.map(_.errorLines).getOrElse(ArrayBuffer.empty[String])) ++
- ("\nYARN Diagnostics: " +: yarnDiagnostics)
-
- override def kill(): Unit = synchronized {
- if (isRunning) {
- try {
- val timeout = SparkYarnApp.getYarnTagToAppIdTimeout(livyConf)
- yarnClient.killApplication(Await.result(appIdPromise.future, timeout))
- } catch {
- // We cannot kill the YARN app without the app id.
- // There's a chance the YARN app hasn't been submitted during a livy-server failure.
- // We don't want a stuck session that can't be deleted. Emit a warning and move on.
- case _: TimeoutException | _: InterruptedException =>
- warn("Deleting a session while its YARN application is not found.")
- yarnAppMonitorThread.interrupt()
- } finally {
- process.foreach(_.destroy())
- }
- }
- }
-
- private def changeState(newState: SparkApp.State.Value): Unit = {
- if (state != newState) {
- listener.foreach(_.stateChanged(state, newState))
- state = newState
- }
- }
-
- /**
- * Find the corresponding YARN application id from an application tag.
- *
- * @param appTag The application tag tagged on the target application.
- * If the tag is not unique, it returns the first application it found.
- * It will be converted to lower case to match YARN's behaviour.
- * @return ApplicationId or the failure.
- */
- @tailrec
- private def getAppIdFromTag(
- appTag: String,
- pollInterval: Duration,
- deadline: Deadline): ApplicationId = {
- val appTagLowerCase = appTag.toLowerCase()
-
- // FIXME Should not loop thru all YARN applications but YarnClient doesn't offer an API.
- // Consider calling rmClient in YarnClient directly.
- yarnClient.getApplications(appType).asScala.find(_.getApplicationTags.contains(appTagLowerCase))
- match {
- case Some(app) => app.getApplicationId
- case None =>
- if (deadline.isOverdue) {
- process.foreach(_.destroy())
- leakedAppTags.put(appTag, System.currentTimeMillis())
- throw new Exception(s"No YARN application is found with tag $appTagLowerCase in " +
- livyConf.getTimeAsMs(LivyConf.YARN_APP_LOOKUP_TIMEOUT)/1000 + " seconds. " +
- "Please check your cluster status, it is may be very busy.")
- } else {
- Clock.sleep(pollInterval.toMillis)
- getAppIdFromTag(appTagLowerCase, pollInterval, deadline)
- }
- }
- }
-
- private def getYarnDiagnostics(appReport: ApplicationReport): IndexedSeq[String] = {
- Option(appReport.getDiagnostics)
- .filter(_.nonEmpty)
- .map[IndexedSeq[String]]("YARN Diagnostics:" +: _.split("\n"))
- .getOrElse(IndexedSeq.empty)
- }
-
- private def isRunning: Boolean = {
- state != SparkApp.State.FAILED && state != SparkApp.State.FINISHED &&
- state != SparkApp.State.KILLED
- }
-
- // Exposed for unit test.
- private[utils] def mapYarnState(
- appId: ApplicationId,
- yarnAppState: YarnApplicationState,
- finalAppStatus: FinalApplicationStatus): SparkApp.State.Value = {
- yarnAppState match {
- case (YarnApplicationState.NEW |
- YarnApplicationState.NEW_SAVING |
- YarnApplicationState.SUBMITTED |
- YarnApplicationState.ACCEPTED) => SparkApp.State.STARTING
- case YarnApplicationState.RUNNING => SparkApp.State.RUNNING
- case YarnApplicationState.FINISHED =>
- finalAppStatus match {
- case FinalApplicationStatus.SUCCEEDED => SparkApp.State.FINISHED
- case FinalApplicationStatus.FAILED => SparkApp.State.FAILED
- case FinalApplicationStatus.KILLED => SparkApp.State.KILLED
- case s =>
- error(s"Unknown YARN final status $appId $s")
- SparkApp.State.FAILED
- }
- case YarnApplicationState.FAILED => SparkApp.State.FAILED
- case YarnApplicationState.KILLED => SparkApp.State.KILLED
- }
- }
-
- // Exposed for unit test.
- // TODO Instead of spawning a thread for every session, create a centralized thread and
- // batch YARN queries.
- private[utils] val yarnAppMonitorThread = Utils.startDaemonThread(s"yarnAppMonitorThread-$this") {
- try {
- // Wait for spark-submit to finish submitting the app to YARN.
- process.foreach { p =>
- val exitCode = p.waitFor()
- if (exitCode != 0) {
- throw new Exception(s"spark-submit exited with code $exitCode}.\n" +
- s"${process.get.inputLines.mkString("\n")}")
- }
- }
-
- // If appId is not known, query YARN by appTag to get it.
- val appId = try {
- appIdOption.map(ConverterUtils.toApplicationId).getOrElse {
- val pollInterval = getYarnPollInterval(livyConf)
- val deadline = getYarnTagToAppIdTimeout(livyConf).fromNow
- getAppIdFromTag(appTag, pollInterval, deadline)
- }
- } catch {
- case e: Exception =>
- appIdPromise.failure(e)
- throw e
- }
- appIdPromise.success(appId)
-
- Thread.currentThread().setName(s"yarnAppMonitorThread-$appId")
- listener.foreach(_.appIdKnown(appId.toString))
-
- val pollInterval = SparkYarnApp.getYarnPollInterval(livyConf)
- var appInfo = AppInfo()
- while (isRunning) {
- try {
- Clock.sleep(pollInterval.toMillis)
-
- // Refresh application state
- val appReport = yarnClient.getApplicationReport(appId)
- yarnDiagnostics = getYarnDiagnostics(appReport)
- changeState(mapYarnState(
- appReport.getApplicationId,
- appReport.getYarnApplicationState,
- appReport.getFinalApplicationStatus))
-
- val latestAppInfo = {
- val attempt =
- yarnClient.getApplicationAttemptReport(appReport.getCurrentApplicationAttemptId)
- val driverLogUrl =
- Try(yarnClient.getContainerReport(attempt.getAMContainerId).getLogUrl)
- .toOption
- AppInfo(driverLogUrl, Option(appReport.getTrackingUrl))
- }
-
- if (appInfo != latestAppInfo) {
- listener.foreach(_.infoChanged(latestAppInfo))
- appInfo = latestAppInfo
- }
- } catch {
- // This exception might be thrown during app is starting up. It's transient.
- case e: ApplicationAttemptNotFoundException =>
- // Workaround YARN-4411: No enum constant FINAL_SAVING from getApplicationAttemptReport()
- case e: IllegalArgumentException =>
- if (e.getMessage.contains("FINAL_SAVING")) {
- debug("Encountered YARN-4411.")
- } else {
- throw e
- }
- }
- }
-
- debug(s"$appId $state ${yarnDiagnostics.mkString(" ")}")
- } catch {
- case e: InterruptedException =>
- yarnDiagnostics = ArrayBuffer("Session stopped by user.")
- changeState(SparkApp.State.KILLED)
- case e: Throwable =>
- error(s"Error whiling refreshing YARN state: $e")
- yarnDiagnostics = ArrayBuffer(e.toString, e.getStackTrace().mkString(" "))
- changeState(SparkApp.State.FAILED)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/LivyConf.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala
new file mode 100644
index 0000000..a79beb4
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/LivyConf.scala
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy
+
+import java.io.File
+import java.lang.{Boolean => JBoolean, Long => JLong}
+import java.util.{Map => JMap}
+
+import scala.collection.JavaConverters._
+
+import org.apache.hadoop.conf.Configuration
+
+import org.apache.livy.client.common.ClientConf
+import org.apache.livy.client.common.ClientConf.ConfEntry
+import org.apache.livy.client.common.ClientConf.DeprecatedConf
+
+object LivyConf {
+
+ case class Entry(override val key: String, override val dflt: AnyRef) extends ConfEntry
+
+ object Entry {
+ def apply(key: String, dflt: Boolean): Entry = Entry(key, dflt: JBoolean)
+ def apply(key: String, dflt: Int): Entry = Entry(key, dflt: Integer)
+ def apply(key: String, dflt: Long): Entry = Entry(key, dflt: JLong)
+ }
+
+ val TEST_MODE = ClientConf.TEST_MODE
+
+ val SPARK_HOME = Entry("livy.server.spark-home", null)
+ val LIVY_SPARK_MASTER = Entry("livy.spark.master", "local")
+ val LIVY_SPARK_DEPLOY_MODE = Entry("livy.spark.deploy-mode", null)
+
+ // Two configurations to specify Spark and related Scala version. These are internal
+ // configurations will be set by LivyServer and used in session creation. It is not required to
+ // set usually unless running with unofficial Spark + Scala versions
+ // (like Spark 2.0 + Scala 2.10, Spark 1.6 + Scala 2.11)
+ val LIVY_SPARK_SCALA_VERSION = Entry("livy.spark.scala-version", null)
+ val LIVY_SPARK_VERSION = Entry("livy.spark.version", null)
+
+ val SESSION_STAGING_DIR = Entry("livy.session.staging-dir", null)
+ val FILE_UPLOAD_MAX_SIZE = Entry("livy.file.upload.max.size", 100L * 1024 * 1024)
+ val LOCAL_FS_WHITELIST = Entry("livy.file.local-dir-whitelist", null)
+ val ENABLE_HIVE_CONTEXT = Entry("livy.repl.enable-hive-context", false)
+
+ val ENVIRONMENT = Entry("livy.environment", "production")
+
+ val SERVER_HOST = Entry("livy.server.host", "0.0.0.0")
+ val SERVER_PORT = Entry("livy.server.port", 8998)
+
+ val UI_ENABLED = Entry("livy.ui.enabled", true)
+
+ val REQUEST_HEADER_SIZE = Entry("livy.server.request-header.size", 131072)
+ val RESPONSE_HEADER_SIZE = Entry("livy.server.response-header.size", 131072)
+
+ val CSRF_PROTECTION = Entry("livy.server.csrf-protection.enabled", false)
+
+ val IMPERSONATION_ENABLED = Entry("livy.impersonation.enabled", false)
+ val SUPERUSERS = Entry("livy.superusers", null)
+
+ val ACCESS_CONTROL_ENABLED = Entry("livy.server.access-control.enabled", false)
+ val ACCESS_CONTROL_USERS = Entry("livy.server.access-control.users", null)
+
+ val SSL_KEYSTORE = Entry("livy.keystore", null)
+ val SSL_KEYSTORE_PASSWORD = Entry("livy.keystore.password", null)
+ val SSL_KEY_PASSWORD = Entry("livy.key-password", null)
+
+ val AUTH_TYPE = Entry("livy.server.auth.type", null)
+ val AUTH_KERBEROS_PRINCIPAL = Entry("livy.server.auth.kerberos.principal", null)
+ val AUTH_KERBEROS_KEYTAB = Entry("livy.server.auth.kerberos.keytab", null)
+ val AUTH_KERBEROS_NAME_RULES = Entry("livy.server.auth.kerberos.name-rules", "DEFAULT")
+
+ val HEARTBEAT_WATCHDOG_INTERVAL = Entry("livy.server.heartbeat-watchdog.interval", "1m")
+
+ val LAUNCH_KERBEROS_PRINCIPAL = Entry("livy.server.launch.kerberos.principal", null)
+ val LAUNCH_KERBEROS_KEYTAB = Entry("livy.server.launch.kerberos.keytab", null)
+ val LAUNCH_KERBEROS_REFRESH_INTERVAL = Entry("livy.server.launch.kerberos.refresh-interval", "1h")
+ val KINIT_FAIL_THRESHOLD = Entry("livy.server.launch.kerberos.kinit-fail-threshold", 5)
+
+ /**
+ * Recovery mode of Livy. Possible values:
+ * off: Default. Turn off recovery. Every time Livy shuts down, it stops and forgets all sessions.
+ * recovery: Livy persists session info to the state store. When Livy restarts, it recovers
+ * previous sessions from the state store.
+ * Must set livy.server.recovery.state-store and livy.server.recovery.state-store.url to
+ * configure the state store.
+ */
+ val RECOVERY_MODE = Entry("livy.server.recovery.mode", "off")
+ /**
+ * Where Livy should store state to for recovery. Possible values:
+ * <empty>: Default. State store disabled.
+ * filesystem: Store state on a file system.
+ * zookeeper: Store state in a Zookeeper instance.
+ */
+ val RECOVERY_STATE_STORE = Entry("livy.server.recovery.state-store", null)
+ /**
+ * For filesystem state store, the path of the state store directory. Please don't use a
+ * filesystem that doesn't support atomic rename (e.g. S3). e.g. file:///tmp/livy or hdfs:///.
+ * For zookeeper, the address to the Zookeeper servers. e.g. host1:port1,host2:port2
+ */
+ val RECOVERY_STATE_STORE_URL = Entry("livy.server.recovery.state-store.url", "")
+
+ // If Livy can't find the yarn app within this time, consider it lost.
+ val YARN_APP_LOOKUP_TIMEOUT = Entry("livy.server.yarn.app-lookup-timeout", "60s")
+
+ // How often Livy polls YARN to refresh YARN app state.
+ val YARN_POLL_INTERVAL = Entry("livy.server.yarn.poll-interval", "5s")
+
+ // Days to keep Livy server request logs.
+ val REQUEST_LOG_RETAIN_DAYS = Entry("livy.server.request-log-retain.days", 5)
+
+ // REPL related jars separated with comma.
+ val REPL_JARS = Entry("livy.repl.jars", null)
+ // RSC related jars separated with comma.
+ val RSC_JARS = Entry("livy.rsc.jars", null)
+
+ // How long to check livy session leakage
+ val YARN_APP_LEAKAGE_CHECK_TIMEOUT = Entry("livy.server.yarn.app-leakage.check-timeout", "600s")
+ // how often to check livy session leakage
+ val YARN_APP_LEAKAGE_CHECK_INTERVAL = Entry("livy.server.yarn.app-leakage.check-interval", "60s")
+
+ // Whether session timeout should be checked, by default it will be checked, which means inactive
+ // session will be stopped after "livy.server.session.timeout"
+ val SESSION_TIMEOUT_CHECK = Entry("livy.server.session.timeout-check", true)
+ // How long will an inactive session be gc-ed.
+ val SESSION_TIMEOUT = Entry("livy.server.session.timeout", "1h")
+ // How long a finished session state will be kept in memory
+ val SESSION_STATE_RETAIN_TIME = Entry("livy.server.session.state-retain.sec", "600s")
+
+ val SPARK_MASTER = "spark.master"
+ val SPARK_DEPLOY_MODE = "spark.submit.deployMode"
+ val SPARK_JARS = "spark.jars"
+ val SPARK_FILES = "spark.files"
+ val SPARK_ARCHIVES = "spark.yarn.dist.archives"
+ val SPARK_PY_FILES = "spark.submit.pyFiles"
+
+ /**
+ * These are Spark configurations that contain lists of files that the user can add to
+ * their jobs in one way or another. Livy needs to pre-process these to make sure the
+ * user can read them (in case they reference local files), and to provide correct URIs
+ * to Spark based on the Livy config.
+ *
+ * The configuration allows adding new configurations in case we either forget something in
+ * the hardcoded list, or new versions of Spark add new configs.
+ */
+ val SPARK_FILE_LISTS = Entry("livy.spark.file-list-configs", null)
+
+ private val HARDCODED_SPARK_FILE_LISTS = Seq(
+ SPARK_JARS,
+ SPARK_FILES,
+ SPARK_ARCHIVES,
+ SPARK_PY_FILES,
+ "spark.yarn.archive",
+ "spark.yarn.dist.files",
+ "spark.yarn.dist.jars",
+ "spark.yarn.jar",
+ "spark.yarn.jars"
+ )
+
+ case class DepConf(
+ override val key: String,
+ override val version: String,
+ override val deprecationMessage: String = "")
+ extends DeprecatedConf
+
+ private val configsWithAlternatives: Map[String, DeprecatedConf] = Map[String, DepConf](
+ LIVY_SPARK_DEPLOY_MODE.key -> DepConf("livy.spark.deployMode", "0.4"),
+ LIVY_SPARK_SCALA_VERSION.key -> DepConf("livy.spark.scalaVersion", "0.4"),
+ ENABLE_HIVE_CONTEXT.key -> DepConf("livy.repl.enableHiveContext", "0.4"),
+ CSRF_PROTECTION.key -> DepConf("livy.server.csrf_protection.enabled", "0.4"),
+ ACCESS_CONTROL_ENABLED.key -> DepConf("livy.server.access_control.enabled", "0.4"),
+ ACCESS_CONTROL_USERS.key -> DepConf("livy.server.access_control.users", "0.4"),
+ AUTH_KERBEROS_NAME_RULES.key -> DepConf("livy.server.auth.kerberos.name_rules", "0.4"),
+ LAUNCH_KERBEROS_REFRESH_INTERVAL.key ->
+ DepConf("livy.server.launch.kerberos.refresh_interval", "0.4"),
+ KINIT_FAIL_THRESHOLD.key -> DepConf("livy.server.launch.kerberos.kinit_fail_threshold", "0.4"),
+ YARN_APP_LEAKAGE_CHECK_TIMEOUT.key ->
+ DepConf("livy.server.yarn.app-leakage.check_timeout", "0.4"),
+ YARN_APP_LEAKAGE_CHECK_INTERVAL.key ->
+ DepConf("livy.server.yarn.app-leakage.check_interval", "0.4")
+ )
+
+ private val deprecatedConfigs: Map[String, DeprecatedConf] = {
+ val configs: Seq[DepConf] = Seq(
+ // There are no deprecated configs without alternatives currently.
+ )
+
+ Map(configs.map { cfg => (cfg.key -> cfg) }: _*)
+ }
+
+}
+
+/**
+ *
+ * @param loadDefaults whether to also load values from the Java system properties
+ */
+class LivyConf(loadDefaults: Boolean) extends ClientConf[LivyConf](null) {
+
+ import LivyConf._
+
+ private lazy val _superusers = configToSeq(SUPERUSERS)
+ private lazy val _allowedUsers = configToSeq(ACCESS_CONTROL_USERS).toSet
+
+ lazy val hadoopConf = new Configuration()
+ lazy val localFsWhitelist = configToSeq(LOCAL_FS_WHITELIST).map { path =>
+ // Make sure the path ends with a single separator.
+ path.stripSuffix("/") + "/"
+ }
+
+ lazy val sparkFileLists = HARDCODED_SPARK_FILE_LISTS ++ configToSeq(SPARK_FILE_LISTS)
+
+ /**
+ * Create a LivyConf that loads defaults from the system properties and the classpath.
+ * @return
+ */
+ def this() = this(true)
+
+ if (loadDefaults) {
+ loadFromMap(sys.props)
+ }
+
+ def loadFromFile(name: String): LivyConf = {
+ getConfigFile(name)
+ .map(Utils.getPropertiesFromFile)
+ .foreach(loadFromMap)
+ this
+ }
+
+ /** Return true if spark master starts with yarn. */
+ def isRunningOnYarn(): Boolean = sparkMaster().startsWith("yarn")
+
+ /** Return the spark deploy mode Livy sessions should use. */
+ def sparkDeployMode(): Option[String] = Option(get(LIVY_SPARK_DEPLOY_MODE)).filterNot(_.isEmpty)
+
+ /** Return the location of the spark home directory */
+ def sparkHome(): Option[String] = Option(get(SPARK_HOME)).orElse(sys.env.get("SPARK_HOME"))
+
+ /** Return the spark master Livy sessions should use. */
+ def sparkMaster(): String = get(LIVY_SPARK_MASTER)
+
+ /** Return the path to the spark-submit executable. */
+ def sparkSubmit(): String = {
+ sparkHome().map { _ + File.separator + "bin" + File.separator + "spark-submit" }.get
+ }
+
+ /** Return the list of superusers. */
+ def superusers(): Seq[String] = _superusers
+
+ /** Return the set of users allowed to use Livy via SPNEGO. */
+ def allowedUsers(): Set[String] = _allowedUsers
+
+ private val configDir: Option[File] = {
+ sys.env.get("LIVY_CONF_DIR")
+ .orElse(sys.env.get("LIVY_HOME").map(path => s"$path${File.separator}conf"))
+ .map(new File(_))
+ .filter(_.exists())
+ }
+
+ private def getConfigFile(name: String): Option[File] = {
+ configDir.map(new File(_, name)).filter(_.exists())
+ }
+
+ private def loadFromMap(map: Iterable[(String, String)]): Unit = {
+ map.foreach { case (k, v) =>
+ if (k.startsWith("livy.")) {
+ set(k, v)
+ }
+ }
+ }
+
+ private def configToSeq(entry: LivyConf.Entry): Seq[String] = {
+ Option(get(entry)).map(_.split("[, ]+").toSeq).getOrElse(Nil)
+ }
+
+ override def getConfigsWithAlternatives: JMap[String, DeprecatedConf] = {
+ configsWithAlternatives.asJava
+ }
+
+ override def getDeprecatedConfigs: JMap[String, DeprecatedConf] = {
+ deprecatedConfigs.asJava
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/package.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/package.scala b/server/src/main/scala/org/apache/livy/package.scala
new file mode 100644
index 0000000..c1a8f28
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/package.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache
+
+import java.util.Properties
+
+import scala.util.control.NonFatal
+
+package object livy {
+
+ private object LivyBuildInfo {
+ val (
+ livyVersion: String,
+ livyBuildUser: String,
+ livyRevision: String,
+ livyBranch: String,
+ livyBuildDate: String,
+ livyRepo: String
+ ) = {
+ val unknown = "<unknown>"
+ val defaultValue = (unknown, unknown, unknown, unknown, unknown, unknown)
+ val resource = Option(Thread.currentThread().getContextClassLoader
+ .getResourceAsStream("livy-version-info.properties"))
+
+ try {
+ resource.map { r =>
+ val properties = new Properties()
+ properties.load(r)
+ (
+ properties.getProperty("version", unknown),
+ properties.getProperty("user", unknown),
+ properties.getProperty("revision", unknown),
+ properties.getProperty("branch", unknown),
+ properties.getProperty("date", unknown),
+ properties.getProperty("url", unknown)
+ )
+ }.getOrElse(defaultValue)
+ } catch {
+ case NonFatal(e) =>
+ // swallow the exception
+ defaultValue
+ } finally {
+ try {
+ resource.foreach(_.close())
+ } catch {
+ case NonFatal(e) => // swallow the exception in closing the stream
+ }
+ }
+ }
+ }
+
+ val LIVY_VERSION = LivyBuildInfo.livyVersion
+ val LIVY_BUILD_USER = LivyBuildInfo.livyBuildUser
+ val LIVY_REVISION = LivyBuildInfo.livyRevision
+ val LIVY_BRANCH = LivyBuildInfo.livyBranch
+ val LIVY_BUILD_DATE = LivyBuildInfo.livyBuildDate
+ val LIVY_REPO_URL = LivyBuildInfo.livyRepo
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/AccessFilter.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/AccessFilter.scala b/server/src/main/scala/org/apache/livy/server/AccessFilter.scala
new file mode 100644
index 0000000..61aebcd
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/AccessFilter.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import javax.servlet._
+import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
+
+import org.apache.livy.LivyConf
+
+class AccessFilter(livyConf: LivyConf) extends Filter {
+
+ override def init(filterConfig: FilterConfig): Unit = {}
+
+ override def doFilter(request: ServletRequest,
+ response: ServletResponse,
+ chain: FilterChain): Unit = {
+ val httpRequest = request.asInstanceOf[HttpServletRequest]
+ val remoteUser = httpRequest.getRemoteUser
+ if (livyConf.allowedUsers.contains(remoteUser)) {
+ chain.doFilter(request, response)
+ } else {
+ val httpServletResponse = response.asInstanceOf[HttpServletResponse]
+ httpServletResponse.sendError(HttpServletResponse.SC_UNAUTHORIZED,
+ "User not authorised to use Livy.")
+ }
+ }
+
+ override def destroy(): Unit = {}
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/ApiVersioningSupport.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/ApiVersioningSupport.scala b/server/src/main/scala/org/apache/livy/server/ApiVersioningSupport.scala
new file mode 100644
index 0000000..c3c52ab
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/ApiVersioningSupport.scala
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import javax.servlet.http.HttpServletRequest
+
+import org.scalatra.{NotAcceptable, ScalatraBase}
+
+/**
+ * Livy's servlets can mix-in this trait to get API version support.
+ *
+ * Example: {{{
+ * import ApiVersions._
+ * class FooServlet
+ * ...
+ * with ApiVersioningSupport
+ * ...
+ * {
+ * get("/test") {
+ * ...
+ * }
+ * get("/test", apiVersion <= v0_2) {
+ * ...
+ * }
+ * get("/test", apiVersion <= v0_1) {
+ * ...
+ * }
+ * }
+ * }}}
+ */
+trait ApiVersioningSupport extends AbstractApiVersioningSupport {
+ this: ScalatraBase =>
+ // Link the abstract trait to Livy's version enum.
+ override val apiVersions = ApiVersions
+ override type ApiVersionType = ApiVersions.Value
+}
+
+trait AbstractApiVersioningSupport {
+ this: ScalatraBase =>
+ protected val apiVersions: Enumeration
+ protected type ApiVersionType
+
+ /**
+ * Before proceeding with routing, validate the specified API version in the request.
+ * If validation passes, cache the parsed API version as a per-request attribute.
+ */
+ before() {
+ request(AbstractApiVersioningSupport.ApiVersionKey) = request.getHeader("Accept") match {
+ case acceptHeader @ AbstractApiVersioningSupport.AcceptHeaderRegex(apiVersion) =>
+ try {
+ apiVersions.withName(apiVersion).asInstanceOf[ApiVersionType]
+ } catch {
+ case e: NoSuchElementException =>
+ halt(NotAcceptable(e.getMessage))
+ }
+ case _ =>
+ // Return the latest version.
+ apiVersions.apply(apiVersions.maxId - 1).asInstanceOf[ApiVersionType]
+ }
+ }
+
+ /**
+ * @return The specified API version in the request.
+ */
+ def apiVersion: ApiVersionType = {
+ request(AbstractApiVersioningSupport.ApiVersionKey).asInstanceOf[ApiVersionType]
+ }
+
+}
+
+object AbstractApiVersioningSupport {
+ // Get every character after "application/vnd.livy.v" until hitting a + sign.
+ private final val AcceptHeaderRegex = """application/vnd\.livy\.v([^\+]*).*""".r
+
+ // AbstractApiVersioningSupport uses a per-request attribute to store the parsed API version.
+ // This is the key name for the attribute.
+ private final val ApiVersionKey = "apiVersion"
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/ApiVersions.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/ApiVersions.scala b/server/src/main/scala/org/apache/livy/server/ApiVersions.scala
new file mode 100644
index 0000000..3d8cf7e
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/ApiVersions.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+/**
+ * This enum defines Livy's API versions.
+ * [[com.cloudera.livy.server.AbstractApiVersioningSupport]] uses this for API version checking.
+ *
+ * Version is defined as <major version>.<minor version>.
+ * When making backward compatible change (adding methods/fields), bump minor version.
+ * When making backward incompatible change (renaming/removing methods/fields), bump major version.
+ * This ensures our users can safely migrate to a newer API version if major version is unchanged.
+ */
+object ApiVersions extends Enumeration {
+ type ApiVersions = Value
+ // ApiVersions are ordered and the ordering is defined by the order of Value() calls.
+ // Please make sure API version is defined in ascending order (Older API before newer).
+ // AbstractApiVersioningSupport relies on the ordering.
+ val v0_1 = Value("0.1")
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/CsrfFilter.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/CsrfFilter.scala b/server/src/main/scala/org/apache/livy/server/CsrfFilter.scala
new file mode 100644
index 0000000..770c8f7
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/CsrfFilter.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.livy.server
+
+import javax.servlet._
+import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
+
+class CsrfFilter extends Filter {
+
+ val METHODS_TO_IGNORE = Set("GET", "OPTIONS", "HEAD");
+
+ val HEADER_NAME = "X-Requested-By";
+
+ override def init(filterConfig: FilterConfig): Unit = {}
+
+ override def doFilter(request: ServletRequest,
+ response: ServletResponse,
+ chain: FilterChain): Unit = {
+ val httpRequest = request.asInstanceOf[HttpServletRequest]
+
+ if (!METHODS_TO_IGNORE.contains(httpRequest.getMethod)
+ && httpRequest.getHeader(HEADER_NAME) == null) {
+ response.asInstanceOf[HttpServletResponse].sendError(HttpServletResponse.SC_BAD_REQUEST,
+ "Missing Required Header for CSRF protection.")
+ } else {
+ chain.doFilter(request, response)
+ }
+ }
+
+ override def destroy(): Unit = {}
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/JsonServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/JsonServlet.scala b/server/src/main/scala/org/apache/livy/server/JsonServlet.scala
new file mode 100644
index 0000000..666a905
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/JsonServlet.scala
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
+
+import scala.concurrent.ExecutionContext
+import scala.reflect.ClassTag
+
+import com.fasterxml.jackson.core.JsonParseException
+import com.fasterxml.jackson.databind.{JsonMappingException, ObjectMapper}
+import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
+import org.scalatra._
+
+/**
+ * An abstract servlet that provides overridden implementations for "post", "put" and "patch"
+ * that can deserialize JSON data directly into user-defined types, without having to go through
+ * a json4s intermediate. Results are also automatically serialized into JSON if the content type
+ * says so.
+ *
+ * Serialization and deserialization are done through Jackson directly, so all Jackson features
+ * are available.
+ */
+abstract class JsonServlet extends ScalatraServlet with ApiFormats with FutureSupport {
+
+ override protected implicit def executor: ExecutionContext = ExecutionContext.global
+
+ private lazy val _defaultMapper = new ObjectMapper()
+ .registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
+
+ /**
+ * Override this method if you need a custom Jackson object mapper; the default mapper
+ * has the default configuration, plus the Scala module.
+ */
+ protected def createMapper(): ObjectMapper = _defaultMapper
+
+ protected final val mapper = createMapper()
+
+ before() {
+ contentType = formats("json")
+ }
+
+ error {
+ case e: JsonParseException => BadRequest(e.getMessage)
+ case e: UnrecognizedPropertyException => BadRequest(e.getMessage)
+ case e: JsonMappingException => BadRequest(e.getMessage)
+ case e =>
+ SessionServlet.error("internal error", e)
+ InternalServerError(e.toString)
+ }
+
+ protected def jpatch[T: ClassTag](t: RouteTransformer*)(action: T => Any): Route = {
+ patch(t: _*) {
+ doAction(request, action)
+ }
+ }
+
+ protected def jpost[T: ClassTag](t: RouteTransformer*)(action: T => Any): Route = {
+ post(t: _*) {
+ doAction(request, action)
+ }
+ }
+
+ protected def jput[T: ClassTag](t: RouteTransformer*)(action: T => Any): Route = {
+ put(t: _*) {
+ doAction(request, action)
+ }
+ }
+
+ override protected def renderResponseBody(actionResult: Any): Unit = {
+ val result = actionResult match {
+ case ActionResult(status, body, headers) if format == "json" =>
+ ActionResult(status, toJson(body), headers)
+ case str: String if format == "json" =>
+ // This should be changed when we implement LIVY-54. For now, just create a dummy
+ // JSON object when a raw string is being returned.
+ toJson(Map("msg" -> str))
+ case other if format == "json" =>
+ toJson(other)
+ case other =>
+ other
+ }
+ super.renderResponseBody(result)
+ }
+
+ protected def bodyAs[T: ClassTag](req: HttpServletRequest)
+ (implicit klass: ClassTag[T]): T = {
+ bodyAs(req, klass.runtimeClass)
+ }
+
+ private def bodyAs[T](req: HttpServletRequest, klass: Class[_]): T = {
+ mapper.readValue(req.getInputStream(), klass).asInstanceOf[T]
+ }
+
+ private def doAction[T: ClassTag](
+ req: HttpServletRequest,
+ action: T => Any)(implicit klass: ClassTag[T]): Any = {
+ action(bodyAs[T](req, klass.runtimeClass))
+ }
+
+ private def isJson(res: HttpServletResponse, headers: Map[String, String] = Map()): Boolean = {
+ val ctypeHeader = "Content-Type"
+ headers.get(ctypeHeader).orElse(Option(res.getHeader(ctypeHeader)))
+ .map(_.startsWith("application/json")).getOrElse(false)
+ }
+
+ private def toResult(obj: Any, res: HttpServletResponse): Any = obj match {
+ case async: AsyncResult =>
+ new AsyncResult {
+ val is = async.is.map(toResult(_, res))
+ }
+ case ActionResult(status, body, headers) if isJson(res, headers) =>
+ ActionResult(status, toJson(body), headers)
+ case body if isJson(res) =>
+ Ok(toJson(body))
+ case other =>
+ other
+ }
+
+ private def toJson(obj: Any): Any = {
+ if (obj != null && obj != ()) {
+ mapper.writeValueAsBytes(obj)
+ } else {
+ null
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/LivyServer.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/LivyServer.scala b/server/src/main/scala/org/apache/livy/server/LivyServer.scala
new file mode 100644
index 0000000..c35e455
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/LivyServer.scala
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import java.io.{BufferedInputStream, InputStream}
+import java.util.concurrent._
+import java.util.EnumSet
+import javax.servlet._
+
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
+
+import org.apache.hadoop.security.{SecurityUtil, UserGroupInformation}
+import org.apache.hadoop.security.authentication.server._
+import org.eclipse.jetty.servlet.FilterHolder
+import org.scalatra.{NotFound, ScalatraServlet}
+import org.scalatra.metrics.MetricsBootstrap
+import org.scalatra.metrics.MetricsSupportExtensions._
+import org.scalatra.servlet.{MultipartConfig, ServletApiImplicits}
+
+import org.apache.livy._
+import org.apache.livy.server.batch.BatchSessionServlet
+import org.apache.livy.server.interactive.InteractiveSessionServlet
+import org.apache.livy.server.recovery.{SessionStore, StateStore}
+import org.apache.livy.server.ui.UIServlet
+import org.apache.livy.sessions.{BatchSessionManager, InteractiveSessionManager}
+import org.apache.livy.sessions.SessionManager.SESSION_RECOVERY_MODE_OFF
+import org.apache.livy.utils.LivySparkUtils._
+import org.apache.livy.utils.SparkYarnApp
+
+class LivyServer extends Logging {
+
+ import LivyConf._
+
+ private var server: WebServer = _
+ private var _serverUrl: Option[String] = None
+ // make livyConf accessible for testing
+ private[livy] var livyConf: LivyConf = _
+
+ private var kinitFailCount: Int = 0
+ private var executor: ScheduledExecutorService = _
+
+ def start(): Unit = {
+ livyConf = new LivyConf().loadFromFile("livy.conf")
+
+ val host = livyConf.get(SERVER_HOST)
+ val port = livyConf.getInt(SERVER_PORT)
+ val multipartConfig = MultipartConfig(
+ maxFileSize = Some(livyConf.getLong(LivyConf.FILE_UPLOAD_MAX_SIZE))
+ ).toMultipartConfigElement
+
+ // Make sure the `spark-submit` program exists, otherwise much of livy won't work.
+ testSparkHome(livyConf)
+
+ // Test spark-submit and get Spark Scala version accordingly.
+ val (sparkVersion, scalaVersionFromSparkSubmit) = sparkSubmitVersion(livyConf)
+ testSparkVersion(sparkVersion)
+
+ // If Spark and Scala version is set manually, should verify if they're consistent with
+ // ones parsed from "spark-submit --version"
+ val formattedSparkVersion = formatSparkVersion(sparkVersion)
+ Option(livyConf.get(LIVY_SPARK_VERSION)).map(formatSparkVersion).foreach { version =>
+ require(formattedSparkVersion == version,
+ s"Configured Spark version $version is not equal to Spark version $formattedSparkVersion " +
+ "got from spark-submit -version")
+ }
+
+ // Set formatted Spark and Scala version into livy configuration, this will be used by
+ // session creation.
+ // TODO Create a new class to pass variables from LivyServer to sessions and remove these
+ // internal LivyConfs.
+ livyConf.set(LIVY_SPARK_VERSION.key, formattedSparkVersion.productIterator.mkString("."))
+ livyConf.set(LIVY_SPARK_SCALA_VERSION.key,
+ sparkScalaVersion(formattedSparkVersion, scalaVersionFromSparkSubmit, livyConf))
+
+ if (UserGroupInformation.isSecurityEnabled) {
+ // If Hadoop security is enabled, run kinit periodically. runKinit() should be called
+ // before any Hadoop operation, otherwise Kerberos exception will be thrown.
+ executor = Executors.newScheduledThreadPool(1,
+ new ThreadFactory() {
+ override def newThread(r: Runnable): Thread = {
+ val thread = new Thread(r)
+ thread.setName("kinit-thread")
+ thread.setDaemon(true)
+ thread
+ }
+ }
+ )
+ val launch_keytab = livyConf.get(LAUNCH_KERBEROS_KEYTAB)
+ val launch_principal = SecurityUtil.getServerPrincipal(
+ livyConf.get(LAUNCH_KERBEROS_PRINCIPAL), host)
+ require(launch_keytab != null,
+ s"Kerberos requires ${LAUNCH_KERBEROS_KEYTAB.key} to be provided.")
+ require(launch_principal != null,
+ s"Kerberos requires ${LAUNCH_KERBEROS_PRINCIPAL.key} to be provided.")
+ if (!runKinit(launch_keytab, launch_principal)) {
+ error("Failed to run kinit, stopping the server.")
+ sys.exit(1)
+ }
+ startKinitThread(launch_keytab, launch_principal)
+ }
+
+ testRecovery(livyConf)
+
+ // Initialize YarnClient ASAP to save time.
+ if (livyConf.isRunningOnYarn()) {
+ SparkYarnApp.init(livyConf)
+ Future { SparkYarnApp.yarnClient }
+ }
+
+ StateStore.init(livyConf)
+ val sessionStore = new SessionStore(livyConf)
+ val batchSessionManager = new BatchSessionManager(livyConf, sessionStore)
+ val interactiveSessionManager = new InteractiveSessionManager(livyConf, sessionStore)
+
+ server = new WebServer(livyConf, host, port)
+ server.context.setResourceBase("src/main/com/cloudera/livy/server")
+
+ val livyVersionServlet = new JsonServlet {
+ before() { contentType = "application/json" }
+
+ get("/") {
+ Map("version" -> LIVY_VERSION,
+ "user" -> LIVY_BUILD_USER,
+ "revision" -> LIVY_REVISION,
+ "branch" -> LIVY_BRANCH,
+ "date" -> LIVY_BUILD_DATE,
+ "url" -> LIVY_REPO_URL)
+ }
+ }
+
+ // Servlet for hosting static files such as html, css, and js
+ // Necessary since Jetty cannot set it's resource base inside a jar
+ // Returns 404 if the file does not exist
+ val staticResourceServlet = new ScalatraServlet {
+ get("/*") {
+ val fileName = params("splat")
+ val notFoundMsg = "File not found"
+
+ if (!fileName.isEmpty) {
+ getClass.getResourceAsStream(s"ui/static/$fileName") match {
+ case is: InputStream => new BufferedInputStream(is)
+ case null => NotFound(notFoundMsg)
+ }
+ } else {
+ NotFound(notFoundMsg)
+ }
+ }
+ }
+
+ def uiRedirectServlet(path: String) = new ScalatraServlet {
+ get("/") {
+ redirect(path)
+ }
+ }
+
+ server.context.addEventListener(
+ new ServletContextListener() with MetricsBootstrap with ServletApiImplicits {
+
+ private def mount(sc: ServletContext, servlet: Servlet, mappings: String*): Unit = {
+ val registration = sc.addServlet(servlet.getClass().getName(), servlet)
+ registration.addMapping(mappings: _*)
+ registration.setMultipartConfig(multipartConfig)
+ }
+
+ override def contextDestroyed(sce: ServletContextEvent): Unit = {
+
+ }
+
+ override def contextInitialized(sce: ServletContextEvent): Unit = {
+ try {
+ val context = sce.getServletContext()
+ context.initParameters(org.scalatra.EnvironmentKey) = livyConf.get(ENVIRONMENT)
+
+ val interactiveServlet =
+ new InteractiveSessionServlet(interactiveSessionManager, sessionStore, livyConf)
+ mount(context, interactiveServlet, "/sessions/*")
+
+ val batchServlet = new BatchSessionServlet(batchSessionManager, sessionStore, livyConf)
+ mount(context, batchServlet, "/batches/*")
+
+ if (livyConf.getBoolean(UI_ENABLED)) {
+ val uiServlet = new UIServlet
+ mount(context, uiServlet, "/ui/*")
+ mount(context, staticResourceServlet, "/static/*")
+ mount(context, uiRedirectServlet("/ui/"), "/*")
+ } else {
+ mount(context, uiRedirectServlet("/metrics"), "/*")
+ }
+
+ context.mountMetricsAdminServlet("/metrics")
+
+ mount(context, livyVersionServlet, "/version/*")
+ } catch {
+ case e: Throwable =>
+ error("Exception thrown when initializing server", e)
+ sys.exit(1)
+ }
+ }
+
+ })
+
+ livyConf.get(AUTH_TYPE) match {
+ case authType @ KerberosAuthenticationHandler.TYPE =>
+ val principal = SecurityUtil.getServerPrincipal(livyConf.get(AUTH_KERBEROS_PRINCIPAL),
+ server.host)
+ val keytab = livyConf.get(AUTH_KERBEROS_KEYTAB)
+ require(principal != null,
+ s"Kerberos auth requires ${AUTH_KERBEROS_PRINCIPAL.key} to be provided.")
+ require(keytab != null,
+ s"Kerberos auth requires ${AUTH_KERBEROS_KEYTAB.key} to be provided.")
+
+ val holder = new FilterHolder(new AuthenticationFilter())
+ holder.setInitParameter(AuthenticationFilter.AUTH_TYPE, authType)
+ holder.setInitParameter(KerberosAuthenticationHandler.PRINCIPAL, principal)
+ holder.setInitParameter(KerberosAuthenticationHandler.KEYTAB, keytab)
+ holder.setInitParameter(KerberosAuthenticationHandler.NAME_RULES,
+ livyConf.get(AUTH_KERBEROS_NAME_RULES))
+ server.context.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType]))
+ info(s"SPNEGO auth enabled (principal = $principal)")
+
+ case null =>
+ // Nothing to do.
+
+ case other =>
+ throw new IllegalArgumentException(s"Invalid auth type: $other")
+ }
+
+ if (livyConf.getBoolean(CSRF_PROTECTION)) {
+ info("CSRF protection is enabled.")
+ val csrfHolder = new FilterHolder(new CsrfFilter())
+ server.context.addFilter(csrfHolder, "/*", EnumSet.allOf(classOf[DispatcherType]))
+ }
+
+ if (livyConf.getBoolean(ACCESS_CONTROL_ENABLED)) {
+ if (livyConf.get(AUTH_TYPE) != null) {
+ info("Access control is enabled.")
+ val accessHolder = new FilterHolder(new AccessFilter(livyConf))
+ server.context.addFilter(accessHolder, "/*", EnumSet.allOf(classOf[DispatcherType]))
+ } else {
+ throw new IllegalArgumentException("Access control was requested but could " +
+ "not be enabled, since authentication is disabled.")
+ }
+ }
+
+ server.start()
+
+ Runtime.getRuntime().addShutdownHook(new Thread("Livy Server Shutdown") {
+ override def run(): Unit = {
+ info("Shutting down Livy server.")
+ server.stop()
+ }
+ })
+
+ _serverUrl = Some(s"${server.protocol}://${server.host}:${server.port}")
+ sys.props("livy.server.server-url") = _serverUrl.get
+ }
+
+ def runKinit(keytab: String, principal: String): Boolean = {
+ val commands = Seq("kinit", "-kt", keytab, principal)
+ val proc = new ProcessBuilder(commands: _*).inheritIO().start()
+ proc.waitFor() match {
+ case 0 =>
+ debug("Ran kinit command successfully.")
+ kinitFailCount = 0
+ true
+ case _ =>
+ warn("Fail to run kinit command.")
+ kinitFailCount += 1
+ false
+ }
+ }
+
+ def startKinitThread(keytab: String, principal: String): Unit = {
+ val refreshInterval = livyConf.getTimeAsMs(LAUNCH_KERBEROS_REFRESH_INTERVAL)
+ val kinitFailThreshold = livyConf.getInt(KINIT_FAIL_THRESHOLD)
+ executor.schedule(
+ new Runnable() {
+ override def run(): Unit = {
+ if (runKinit(keytab, principal)) {
+ // schedule another kinit run with a fixed delay.
+ executor.schedule(this, refreshInterval, TimeUnit.MILLISECONDS)
+ } else {
+ // schedule another retry at once or fail the livy server if too many times kinit fail
+ if (kinitFailCount >= kinitFailThreshold) {
+ error(s"Exit LivyServer after ${kinitFailThreshold} times failures running kinit.")
+ if (server.server.isStarted()) {
+ stop()
+ } else {
+ sys.exit(1)
+ }
+ } else {
+ executor.submit(this)
+ }
+ }
+ }
+ }, refreshInterval, TimeUnit.MILLISECONDS)
+ }
+
+ def join(): Unit = server.join()
+
+ def stop(): Unit = {
+ if (server != null) {
+ server.stop()
+ }
+ }
+
+ def serverUrl(): String = {
+ _serverUrl.getOrElse(throw new IllegalStateException("Server not yet started."))
+ }
+
+ private[livy] def testRecovery(livyConf: LivyConf): Unit = {
+ if (!livyConf.isRunningOnYarn()) {
+ // If recovery is turned on but we are not running on YARN, quit.
+ require(livyConf.get(LivyConf.RECOVERY_MODE) == SESSION_RECOVERY_MODE_OFF,
+ "Session recovery requires YARN.")
+ }
+ }
+}
+
+object LivyServer {
+
+ def main(args: Array[String]): Unit = {
+ val server = new LivyServer()
+ try {
+ server.start()
+ server.join()
+ } finally {
+ server.stop()
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/SessionServlet.scala b/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
new file mode 100644
index 0000000..f77171b
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import javax.servlet.http.HttpServletRequest
+
+import org.scalatra._
+import scala.concurrent._
+import scala.concurrent.duration._
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.sessions.{Session, SessionManager}
+import org.apache.livy.sessions.Session.RecoveryMetadata
+
+object SessionServlet extends Logging
+
+/**
+ * Base servlet for session management. All helper methods in this class assume that the session
+ * id parameter in the handler's URI is "id".
+ *
+ * Type parameters:
+ * S: the session type
+ */
+abstract class SessionServlet[S <: Session, R <: RecoveryMetadata](
+ private[livy] val sessionManager: SessionManager[S, R],
+ livyConf: LivyConf)
+ extends JsonServlet
+ with ApiVersioningSupport
+ with MethodOverride
+ with UrlGeneratorSupport
+ with GZipSupport
+{
+ /**
+ * Creates a new session based on the current request. The implementation is responsible for
+ * parsing the body of the request.
+ */
+ protected def createSession(req: HttpServletRequest): S
+
+ /**
+ * Returns a object representing the session data to be sent back to the client.
+ */
+ protected def clientSessionView(session: S, req: HttpServletRequest): Any = session
+
+ override def shutdown(): Unit = {
+ sessionManager.shutdown()
+ }
+
+ before() {
+ contentType = "application/json"
+ }
+
+ get("/") {
+ val from = params.get("from").map(_.toInt).getOrElse(0)
+ val size = params.get("size").map(_.toInt).getOrElse(100)
+
+ val sessions = sessionManager.all()
+
+ Map(
+ "from" -> from,
+ "total" -> sessionManager.size(),
+ "sessions" -> sessions.view(from, from + size).map(clientSessionView(_, request))
+ )
+ }
+
+ val getSession = get("/:id") {
+ withUnprotectedSession { session =>
+ clientSessionView(session, request)
+ }
+ }
+
+ get("/:id/state") {
+ withUnprotectedSession { session =>
+ Map("id" -> session.id, "state" -> session.state.toString)
+ }
+ }
+
+ get("/:id/log") {
+ withSession { session =>
+ val from = params.get("from").map(_.toInt)
+ val size = params.get("size").map(_.toInt)
+ val (from_, total, logLines) = serializeLogs(session, from, size)
+
+ Map(
+ "id" -> session.id,
+ "from" -> from_,
+ "total" -> total,
+ "log" -> logLines)
+ }
+ }
+
+ delete("/:id") {
+ withSession { session =>
+ sessionManager.delete(session.id) match {
+ case Some(future) =>
+ Await.ready(future, Duration.Inf)
+ Ok(Map("msg" -> "deleted"))
+
+ case None =>
+ NotFound(s"Session ${session.id} already stopped.")
+ }
+ }
+ }
+
+ post("/") {
+ val session = sessionManager.register(createSession(request))
+ // Because it may take some time to establish the session, update the last activity
+ // time before returning the session info to the client.
+ session.recordActivity()
+ Created(clientSessionView(session, request),
+ headers = Map("Location" ->
+ (getRequestPathInfo(request) + url(getSession, "id" -> session.id.toString))))
+ }
+
+ private def getRequestPathInfo(request: HttpServletRequest): String = {
+ if (request.getPathInfo != null && request.getPathInfo != "/") {
+ request.getPathInfo
+ } else {
+ ""
+ }
+ }
+
+ error {
+ case e: IllegalArgumentException => BadRequest(e.getMessage)
+ }
+
+ /**
+ * Returns the remote user for the given request. Separate method so that tests can override it.
+ */
+ protected def remoteUser(req: HttpServletRequest): String = req.getRemoteUser()
+
+ /**
+ * Checks that the request's user can impersonate the target user.
+ *
+ * If the user does not have permission to impersonate, then halt execution.
+ *
+ * @return The user that should be impersonated. That can be the target user if defined, the
+ * request's user - which may not be defined - otherwise, or `None` if impersonation is
+ * disabled.
+ */
+ protected def checkImpersonation(
+ target: Option[String],
+ req: HttpServletRequest): Option[String] = {
+ if (livyConf.getBoolean(LivyConf.IMPERSONATION_ENABLED)) {
+ if (!target.map(hasAccess(_, req)).getOrElse(true)) {
+ halt(Forbidden(s"User '${remoteUser(req)}' not allowed to impersonate '$target'."))
+ }
+ target.orElse(Option(remoteUser(req)))
+ } else {
+ None
+ }
+ }
+
+ /**
+ * Check that the request's user has access to resources owned by the given target user.
+ */
+ protected def hasAccess(target: String, req: HttpServletRequest): Boolean = {
+ val user = remoteUser(req)
+ user == null || user == target || livyConf.superusers().contains(user)
+ }
+
+ /**
+ * Performs an operation on the session, without checking for ownership. Operations executed
+ * via this method must not modify the session in any way, or return potentially sensitive
+ * information.
+ */
+ protected def withUnprotectedSession(fn: (S => Any)): Any = doWithSession(fn, true)
+
+ /**
+ * Performs an operation on the session, verifying whether the caller is the owner of the
+ * session.
+ */
+ protected def withSession(fn: (S => Any)): Any = doWithSession(fn, false)
+
+ private def doWithSession(fn: (S => Any), allowAll: Boolean): Any = {
+ val sessionId = params("id").toInt
+ sessionManager.get(sessionId) match {
+ case Some(session) =>
+ if (allowAll || hasAccess(session.owner, request)) {
+ fn(session)
+ } else {
+ Forbidden()
+ }
+ case None =>
+ NotFound(s"Session '$sessionId' not found.")
+ }
+ }
+
+ private def serializeLogs(session: S, fromOpt: Option[Int], sizeOpt: Option[Int]) = {
+ val lines = session.logLines()
+
+ val size = sizeOpt.getOrElse(100)
+ var from = fromOpt.getOrElse(-1)
+ if (from < 0) {
+ from = math.max(0, lines.length - size)
+ }
+ val until = from + size
+
+ (from, lines.length, lines.view(from, until))
+ }
+}
[21/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java b/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java
new file mode 100644
index 0000000..20be563
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java
@@ -0,0 +1,510 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.driver;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import io.netty.channel.ChannelHandler.Sharable;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.util.concurrent.ScheduledFuture;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.client.common.Serializer;
+import org.apache.livy.rsc.BaseProtocol;
+import org.apache.livy.rsc.BypassJobStatus;
+import org.apache.livy.rsc.FutureListener;
+import org.apache.livy.rsc.RSCConf;
+import org.apache.livy.rsc.Utils;
+import org.apache.livy.rsc.rpc.Rpc;
+import org.apache.livy.rsc.rpc.RpcDispatcher;
+import org.apache.livy.rsc.rpc.RpcServer;
+
+import static org.apache.livy.rsc.RSCConf.Entry.*;
+
+/**
+ * Driver code for the Spark client library.
+ */
+@Sharable
+public class RSCDriver extends BaseProtocol {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RSCDriver.class);
+
+ private final Serializer serializer;
+ private final Object jcLock;
+ private final Object shutdownLock;
+ private final ExecutorService executor;
+ private final File localTmpDir;
+ // Used to queue up requests while the SparkContext is being created.
+ private final List<JobWrapper<?>> jobQueue;
+ // Keeps track of connected clients.
+ protected final Collection<Rpc> clients;
+
+ final Map<String, JobWrapper<?>> activeJobs;
+ private final Collection<BypassJobWrapper> bypassJobs;
+
+ private RpcServer server;
+ private volatile JobContextImpl jc;
+ private volatile boolean running;
+
+ protected final SparkConf conf;
+ protected final RSCConf livyConf;
+
+ private final AtomicReference<ScheduledFuture<?>> idleTimeout;
+
+ public RSCDriver(SparkConf conf, RSCConf livyConf) throws Exception {
+ Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwx------");
+ this.localTmpDir = Files.createTempDirectory("rsc-tmp",
+ PosixFilePermissions.asFileAttribute(perms)).toFile();
+ this.executor = Executors.newCachedThreadPool();
+ this.jobQueue = new LinkedList<>();
+ this.clients = new ConcurrentLinkedDeque<>();
+ this.serializer = new Serializer();
+
+ this.conf = conf;
+ this.livyConf = livyConf;
+ this.jcLock = new Object();
+ this.shutdownLock = new Object();
+
+ this.activeJobs = new ConcurrentHashMap<>();
+ this.bypassJobs = new ConcurrentLinkedDeque<>();
+ this.idleTimeout = new AtomicReference<>();
+ }
+
+ private synchronized void shutdown() {
+ if (!running) {
+ return;
+ }
+
+ running = false;
+
+ // Cancel any pending jobs.
+ for (JobWrapper<?> job : activeJobs.values()) {
+ job.cancel();
+ }
+
+ try {
+ shutdownContext();
+ } catch (Exception e) {
+ LOG.warn("Error during shutdown.", e);
+ }
+ try {
+ shutdownServer();
+ } catch (Exception e) {
+ LOG.warn("Error during shutdown.", e);
+ }
+
+ synchronized (shutdownLock) {
+ shutdownLock.notifyAll();
+ }
+ synchronized (jcLock) {
+ jcLock.notifyAll();
+ }
+ }
+
+ private void initializeServer() throws Exception {
+ String clientId = livyConf.get(CLIENT_ID);
+ Utils.checkArgument(clientId != null, "No client ID provided.");
+ String secret = livyConf.get(CLIENT_SECRET);
+ Utils.checkArgument(secret != null, "No secret provided.");
+
+ String launcherAddress = livyConf.get(LAUNCHER_ADDRESS);
+ Utils.checkArgument(launcherAddress != null, "Missing launcher address.");
+ int launcherPort = livyConf.getInt(LAUNCHER_PORT);
+ Utils.checkArgument(launcherPort > 0, "Missing launcher port.");
+
+ LOG.info("Connecting to: {}:{}", launcherAddress, launcherPort);
+
+ // We need to unset this configuration since it doesn't really apply for the driver side.
+ // If the driver runs on a multi-homed machine, this can lead to issues where the Livy
+ // server cannot connect to the auto-detected address, but since the driver can run anywhere
+ // on the cluster, it would be tricky to solve that problem in a generic way.
+ livyConf.set(RPC_SERVER_ADDRESS, null);
+
+ if (livyConf.getBoolean(TEST_STUCK_START_DRIVER)) {
+ // Test flag is turned on so we will just infinite loop here. It should cause
+ // timeout and we should still see yarn application being cleaned up.
+ LOG.info("Infinite looping as test flag TEST_STUCK_START_SESSION is turned on.");
+ while(true) {
+ try {
+ TimeUnit.MINUTES.sleep(10);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted during test sleep.", e);
+ }
+ }
+ }
+
+ // Bring up the RpcServer an register the secret provided by the Livy server as a client.
+ LOG.info("Starting RPC server...");
+ this.server = new RpcServer(livyConf);
+ server.registerClient(clientId, secret, new RpcServer.ClientCallback() {
+ @Override
+ public RpcDispatcher onNewClient(Rpc client) {
+ registerClient(client);
+ return RSCDriver.this;
+ }
+
+ @Override
+ public void onSaslComplete(Rpc client) {
+ onClientAuthenticated(client);
+ }
+ });
+
+ // The RPC library takes care of timing out this.
+ Rpc callbackRpc = Rpc.createClient(livyConf, server.getEventLoopGroup(),
+ launcherAddress, launcherPort, clientId, secret, this).get();
+ try {
+ callbackRpc.call(new RemoteDriverAddress(server.getAddress(), server.getPort())).get(
+ livyConf.getTimeAsMs(RPC_CLIENT_HANDSHAKE_TIMEOUT), TimeUnit.MILLISECONDS);
+ } catch (TimeoutException te) {
+ LOG.warn("Timed out sending address to Livy server, shutting down.");
+ throw te;
+ } finally {
+ callbackRpc.close();
+ }
+
+ // At this point we install the idle timeout handler, in case the Livy server fails to connect
+ // back.
+ setupIdleTimeout();
+ }
+
+ private void registerClient(final Rpc client) {
+ clients.add(client);
+ stopIdleTimeout();
+
+ Utils.addListener(client.getChannel().closeFuture(), new FutureListener<Void>() {
+ @Override
+ public void onSuccess(Void unused) {
+ clients.remove(client);
+ setupIdleTimeout();
+ }
+ });
+ LOG.debug("Registered new connection from {}.", client.getChannel());
+ }
+
+ private void setupIdleTimeout() {
+ if (clients.size() > 0) {
+ return;
+ }
+
+ Runnable timeoutTask = new Runnable() {
+ @Override
+ public void run() {
+ LOG.warn("Shutting down RSC due to idle timeout ({}).", livyConf.get(SERVER_IDLE_TIMEOUT));
+ shutdown();
+ }
+ };
+ ScheduledFuture<?> timeout = server.getEventLoopGroup().schedule(timeoutTask,
+ livyConf.getTimeAsMs(SERVER_IDLE_TIMEOUT), TimeUnit.MILLISECONDS);
+
+ // If there's already an idle task registered, then cancel the new one.
+ if (!this.idleTimeout.compareAndSet(null, timeout)) {
+ LOG.debug("Timeout task already registered.");
+ timeout.cancel(false);
+ }
+
+ // If a new client connected while the idle task was being set up, then stop the task.
+ if (clients.size() > 0) {
+ stopIdleTimeout();
+ }
+ }
+
+ private void stopIdleTimeout() {
+ ScheduledFuture<?> idleTimeout = this.idleTimeout.getAndSet(null);
+ if (idleTimeout != null) {
+ LOG.debug("Cancelling idle timeout since new client connected.");
+ idleTimeout.cancel(false);
+ }
+ }
+
+ protected void broadcast(Object msg) {
+ for (Rpc client : clients) {
+ try {
+ client.call(msg);
+ } catch (Exception e) {
+ LOG.warn("Failed to send message to client " + client, e);
+ }
+ }
+ }
+
+ /**
+ * Initializes the SparkContext used by this driver. This implementation creates a
+ * context with the provided configuration. Subclasses can override this behavior,
+ * and returning a null context is allowed. In that case, the context exposed by
+ * JobContext will be null.
+ */
+ protected JavaSparkContext initializeContext() throws Exception {
+ long t1 = System.nanoTime();
+ LOG.info("Starting Spark context...");
+ JavaSparkContext sc = new JavaSparkContext(conf);
+ LOG.info("Spark context finished initialization in {}ms",
+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t1));
+ return sc;
+ }
+
+ protected void onClientAuthenticated(final Rpc client) {
+
+ }
+
+ /**
+ * Called to shut down the driver; any initialization done by initializeContext() should
+ * be undone here. This is guaranteed to be called only once.
+ */
+ protected void shutdownContext() {
+ if (jc != null) {
+ jc.stop();
+ }
+ executor.shutdownNow();
+ try {
+ FileUtils.deleteDirectory(localTmpDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to delete local tmp dir: " + localTmpDir, e);
+ }
+ }
+
+ private void shutdownServer() {
+ if (server != null) {
+ server.close();
+ }
+ for (Rpc client: clients) {
+ client.close();
+ }
+ }
+
+ void run() throws Exception {
+ this.running = true;
+
+ // Set up a class loader that can be modified, so that we can add jars uploaded
+ // by the client to the driver's class path.
+ ClassLoader driverClassLoader = new MutableClassLoader(
+ Thread.currentThread().getContextClassLoader());
+ Thread.currentThread().setContextClassLoader(driverClassLoader);
+
+ try {
+ initializeServer();
+
+ JavaSparkContext sc = initializeContext();
+ synchronized (jcLock) {
+ jc = new JobContextImpl(sc, localTmpDir, this);
+ jcLock.notifyAll();
+ }
+
+ synchronized (jcLock) {
+ for (JobWrapper<?> job : jobQueue) {
+ submit(job);
+ }
+ jobQueue.clear();
+ }
+
+ synchronized (shutdownLock) {
+ try {
+ while (running) {
+ shutdownLock.wait();
+ }
+ } catch (InterruptedException ie) {
+ // Nothing to do.
+ }
+ }
+ } finally {
+ shutdown();
+ }
+ }
+
+ public void submit(JobWrapper<?> job) {
+ if (jc != null) {
+ job.submit(executor);
+ return;
+ }
+ synchronized (jcLock) {
+ if (jc != null) {
+ job.submit(executor);
+ } else {
+ LOG.info("SparkContext not yet up, queueing job request.");
+ jobQueue.add(job);
+ }
+ }
+ }
+
+ JobContextImpl jobContext() {
+ return jc;
+ }
+
+ Serializer serializer() {
+ return serializer;
+ }
+
+ <T> void jobFinished(String jobId, T result, Throwable error) {
+ LOG.debug("Send job({}) result to Client.", jobId);
+ broadcast(new JobResult<T>(jobId, result, error));
+ }
+
+ void jobStarted(String jobId) {
+ broadcast(new JobStarted(jobId));
+ }
+
+ public void handle(ChannelHandlerContext ctx, CancelJob msg) {
+ JobWrapper<?> job = activeJobs.get(msg.id);
+ if (job == null || !job.cancel()) {
+ LOG.info("Requested to cancel an already finished job.");
+ }
+ }
+
+ public void handle(ChannelHandlerContext ctx, EndSession msg) {
+ if (livyConf.getBoolean(TEST_STUCK_END_SESSION)) {
+ LOG.warn("Ignoring EndSession request because TEST_STUCK_END_SESSION is set.");
+ } else {
+ LOG.debug("Shutting down due to EndSession request.");
+ shutdown();
+ }
+ }
+
+ public void handle(ChannelHandlerContext ctx, JobRequest<?> msg) {
+ LOG.info("Received job request {}", msg.id);
+ JobWrapper<?> wrapper = new JobWrapper<>(this, msg.id, msg.job);
+ activeJobs.put(msg.id, wrapper);
+ submit(wrapper);
+ }
+
+ public void handle(ChannelHandlerContext ctx, BypassJobRequest msg) throws Exception {
+ LOG.info("Received bypass job request {}", msg.id);
+ BypassJobWrapper wrapper = createWrapper(msg);
+ bypassJobs.add(wrapper);
+ activeJobs.put(msg.id, wrapper);
+ if (msg.synchronous) {
+ waitForJobContext();
+ try {
+ wrapper.call();
+ } catch (Throwable t) {
+ // Wrapper already logged and saved the exception, just avoid it bubbling up
+ // to the RPC layer.
+ }
+ } else {
+ submit(wrapper);
+ }
+ }
+
+ protected BypassJobWrapper createWrapper(BypassJobRequest msg) throws Exception {
+ return new BypassJobWrapper(this, msg.id, new BypassJob(this.serializer(), msg.serializedJob));
+ }
+
+ @SuppressWarnings("unchecked")
+ public Object handle(ChannelHandlerContext ctx, SyncJobRequest msg) throws Exception {
+ waitForJobContext();
+ return msg.job.call(jc);
+ }
+
+ public BypassJobStatus handle(ChannelHandlerContext ctx, GetBypassJobStatus msg) {
+ for (Iterator<BypassJobWrapper> it = bypassJobs.iterator(); it.hasNext();) {
+ BypassJobWrapper job = it.next();
+ if (job.jobId.equals(msg.id)) {
+ BypassJobStatus status = job.getStatus();
+ switch (status.state) {
+ case CANCELLED:
+ case FAILED:
+ case SUCCEEDED:
+ it.remove();
+ break;
+
+ default:
+ // No-op.
+ }
+ return status;
+ }
+ }
+
+ throw new NoSuchElementException(msg.id);
+ }
+
+ private void waitForJobContext() throws InterruptedException {
+ synchronized (jcLock) {
+ while (jc == null) {
+ jcLock.wait();
+ if (!running) {
+ throw new IllegalStateException("Remote context is shutting down.");
+ }
+ }
+ }
+ }
+
+ protected void addFile(String path) {
+ jc.sc().addFile(path);
+ }
+
+ protected void addJarOrPyFile(String path) throws Exception {
+ File localCopyDir = new File(jc.getLocalTmpDir(), "__livy__");
+ File localCopy = copyFileToLocal(localCopyDir, path, jc.sc().sc());
+ addLocalFileToClassLoader(localCopy);
+ jc.sc().addJar(path);
+ }
+
+ public void addLocalFileToClassLoader(File localCopy) throws MalformedURLException {
+ MutableClassLoader cl = (MutableClassLoader) Thread.currentThread().getContextClassLoader();
+ cl.addURL(localCopy.toURI().toURL());
+ }
+
+ public File copyFileToLocal(
+ File localCopyDir,
+ String filePath,
+ SparkContext sc) throws Exception {
+ synchronized (jc) {
+ if (!localCopyDir.isDirectory() && !localCopyDir.mkdir()) {
+ throw new IOException("Failed to create directory to add pyFile");
+ }
+ }
+ URI uri = new URI(filePath);
+ String name = uri.getFragment() != null ? uri.getFragment() : uri.getPath();
+ name = new File(name).getName();
+ File localCopy = new File(localCopyDir, name);
+
+ if (localCopy.exists()) {
+ throw new IOException(String.format("A file with name %s has " +
+ "already been uploaded.", name));
+ }
+ Configuration conf = sc.hadoopConfiguration();
+ FileSystem fs = FileSystem.get(uri, conf);
+ fs.copyToLocalFile(new Path(uri), new Path(localCopy.toURI()));
+ return localCopy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriverBootstrapper.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriverBootstrapper.java b/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriverBootstrapper.java
new file mode 100644
index 0000000..09f0ea4
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriverBootstrapper.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.driver;
+
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Properties;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import org.apache.spark.SparkConf;
+
+import org.apache.livy.rsc.RSCConf;
+import static org.apache.livy.rsc.RSCConf.Entry.*;
+
+/**
+ * The entry point for the RSC. Parses command line arguments and instantiates the correct
+ * driver based on the configuration.
+ *
+ * The driver is expected to have a public constructor that takes a two parameters:
+ * a SparkConf and a RSCConf.
+ */
+public final class RSCDriverBootstrapper {
+
+ public static void main(String[] args) throws Exception {
+ Properties props;
+
+ switch (args.length) {
+ case 0:
+ props = System.getProperties();
+ break;
+
+ case 1:
+ props = new Properties();
+ Reader r = new InputStreamReader(new FileInputStream(args[0]), UTF_8);
+ try {
+ props.load(r);
+ } finally {
+ r.close();
+ }
+ break;
+
+ default:
+ throw new IllegalArgumentException("Too many arguments.");
+ }
+
+ SparkConf conf = new SparkConf(false);
+ RSCConf livyConf = new RSCConf(null);
+
+ for (String key : props.stringPropertyNames()) {
+ String value = props.getProperty(key);
+ if (key.startsWith(RSCConf.LIVY_SPARK_PREFIX)) {
+ livyConf.set(key.substring(RSCConf.LIVY_SPARK_PREFIX.length()), value);
+ props.remove(key);
+ } else if (key.startsWith(RSCConf.SPARK_CONF_PREFIX)) {
+ conf.set(key, value);
+ }
+ }
+
+ String driverClass = livyConf.get(DRIVER_CLASS);
+ if (driverClass == null) {
+ driverClass = RSCDriver.class.getName();
+ }
+
+ RSCDriver driver = (RSCDriver) Thread.currentThread()
+ .getContextClassLoader()
+ .loadClass(driverClass)
+ .getConstructor(SparkConf.class, RSCConf.class)
+ .newInstance(conf, livyConf);
+
+ driver.run();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/Statement.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/Statement.java b/rsc/src/main/java/org/apache/livy/rsc/driver/Statement.java
new file mode 100644
index 0000000..d2b7e8f
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/Statement.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.driver;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.fasterxml.jackson.annotation.JsonRawValue;
+
+public class Statement {
+ public final Integer id;
+ public final String code;
+ public final AtomicReference<StatementState> state;
+ @JsonRawValue
+ public volatile String output;
+ public double progress;
+
+ public Statement(Integer id, String code, StatementState state, String output) {
+ this.id = id;
+ this.code = code;
+ this.state = new AtomicReference<>(state);
+ this.output = output;
+ this.progress = 0.0;
+ }
+
+ public Statement() {
+ this(null, null, null, null);
+ }
+
+ public boolean compareAndTransit(final StatementState from, final StatementState to) {
+ if (state.compareAndSet(from, to)) {
+ StatementState.validate(from, to);
+ return true;
+ }
+ return false;
+ }
+
+ public void updateProgress(double p) {
+ if (this.state.get().isOneOf(StatementState.Cancelled, StatementState.Available)) {
+ this.progress = 1.0;
+ } else {
+ this.progress = p;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/StatementState.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/StatementState.java b/rsc/src/main/java/org/apache/livy/rsc/driver/StatementState.java
new file mode 100644
index 0000000..787fc77
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/StatementState.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.driver;
+
+import java.util.*;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public enum StatementState {
+ Waiting("waiting"),
+ Running("running"),
+ Available("available"),
+ Cancelling("cancelling"),
+ Cancelled("cancelled");
+
+ private static final Logger LOG = LoggerFactory.getLogger(StatementState.class);
+
+ private final String state;
+
+ StatementState(final String text) {
+ this.state = text;
+ }
+
+ @JsonValue
+ @Override
+ public String toString() {
+ return state;
+ }
+
+ public boolean isOneOf(StatementState... states) {
+ for (StatementState s : states) {
+ if (s == this) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static final Map<StatementState, List<StatementState>> PREDECESSORS;
+
+ static void put(StatementState key,
+ Map<StatementState, List<StatementState>> map,
+ StatementState... values) {
+ map.put(key, Collections.unmodifiableList(Arrays.asList(values)));
+ }
+
+ static {
+ final Map<StatementState, List<StatementState>> predecessors =
+ new EnumMap<>(StatementState.class);
+ put(Waiting, predecessors);
+ put(Running, predecessors, Waiting);
+ put(Available, predecessors, Running);
+ put(Cancelling, predecessors, Running);
+ put(Cancelled, predecessors, Waiting, Cancelling);
+
+ PREDECESSORS = Collections.unmodifiableMap(predecessors);
+ }
+
+ static boolean isValid(StatementState from, StatementState to) {
+ return PREDECESSORS.get(to).contains(from);
+ }
+
+ static void validate(StatementState from, StatementState to) {
+ LOG.debug("{} -> {}", from, to);
+ if (!isValid(from, to)) {
+ throw new IllegalStateException("Illegal Transition: " + from + " -> " + to);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/rpc/KryoMessageCodec.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/rpc/KryoMessageCodec.java b/rsc/src/main/java/org/apache/livy/rsc/rpc/KryoMessageCodec.java
new file mode 100644
index 0000000..b860e65
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/rpc/KryoMessageCodec.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.rpc;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.ByteBufferInputStream;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import com.esotericsoftware.shaded.org.objenesis.strategy.StdInstantiatorStrategy;
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.ByteToMessageCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.client.common.Serializer;
+import org.apache.livy.rsc.Utils;
+
+/**
+ * Codec that serializes / deserializes objects using Kryo. Objects are encoded with a 4-byte
+ * header with the length of the serialized data.
+ */
+class KryoMessageCodec extends ByteToMessageCodec<Object> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KryoMessageCodec.class);
+
+ private final int maxMessageSize;
+ private final Serializer serializer;
+ private volatile EncryptionHandler encryptionHandler;
+
+ public KryoMessageCodec(int maxMessageSize, Class<?>... messages) {
+ this.maxMessageSize = maxMessageSize;
+ this.serializer = new Serializer(messages);
+ this.encryptionHandler = null;
+ }
+
+ @Override
+ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out)
+ throws Exception {
+ if (in.readableBytes() < 4) {
+ return;
+ }
+
+ in.markReaderIndex();
+ int msgSize = in.readInt();
+ checkSize(msgSize);
+
+ if (in.readableBytes() < msgSize) {
+ // Incomplete message in buffer.
+ in.resetReaderIndex();
+ return;
+ }
+
+ try {
+ ByteBuffer nioBuffer = maybeDecrypt(in.nioBuffer(in.readerIndex(), msgSize));
+ Object msg = serializer.deserialize(nioBuffer);
+ LOG.debug("Decoded message of type {} ({} bytes)",
+ msg != null ? msg.getClass().getName() : msg, msgSize);
+ out.add(msg);
+ } finally {
+ in.skipBytes(msgSize);
+ }
+ }
+
+ @Override
+ protected void encode(ChannelHandlerContext ctx, Object msg, ByteBuf buf)
+ throws Exception {
+ ByteBuffer msgData = maybeEncrypt(serializer.serialize(msg));
+ LOG.debug("Encoded message of type {} ({} bytes)", msg.getClass().getName(),
+ msgData.remaining());
+ checkSize(msgData.remaining());
+
+ buf.ensureWritable(msgData.remaining() + 4);
+ buf.writeInt(msgData.remaining());
+ buf.writeBytes(msgData);
+ }
+
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+ if (encryptionHandler != null) {
+ encryptionHandler.dispose();
+ }
+ super.channelInactive(ctx);
+ }
+
+ private void checkSize(int msgSize) {
+ Utils.checkArgument(msgSize > 0, "Message size (%s bytes) must be positive.", msgSize);
+ Utils.checkArgument(maxMessageSize <= 0 || msgSize <= maxMessageSize,
+ "Message (%s bytes) exceeds maximum allowed size (%s bytes).", msgSize, maxMessageSize);
+ }
+
+ private ByteBuffer maybeEncrypt(ByteBuffer data) throws Exception {
+ return doWrapOrUnWrap(data, true);
+ }
+
+ private ByteBuffer maybeDecrypt(ByteBuffer data) throws Exception {
+ return doWrapOrUnWrap(data, false);
+ }
+
+ private ByteBuffer doWrapOrUnWrap(ByteBuffer data, boolean wrap) throws IOException {
+ if (encryptionHandler == null) {
+ return data;
+ }
+
+ byte[] byteData;
+ int len = data.limit() - data.position();
+ int offset;
+ if (data.hasArray()) {
+ byteData = data.array();
+ offset = data.position() + data.arrayOffset();
+ data.position(data.limit());
+ } else {
+ byteData = new byte[len];
+ offset = 0;
+ data.get(byteData);
+ }
+
+ byte[] result;
+ if (wrap) {
+ result = encryptionHandler.wrap(byteData, offset, len);
+ } else {
+ result = encryptionHandler.unwrap(byteData, offset, len);
+ }
+ return ByteBuffer.wrap(result);
+ }
+
+ void setEncryptionHandler(EncryptionHandler handler) {
+ this.encryptionHandler = handler;
+ }
+
+ interface EncryptionHandler {
+
+ byte[] wrap(byte[] data, int offset, int len) throws IOException;
+
+ byte[] unwrap(byte[] data, int offset, int len) throws IOException;
+
+ void dispose() throws IOException;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/rpc/README.md
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/rpc/README.md b/rsc/src/main/java/org/apache/livy/rsc/rpc/README.md
new file mode 100644
index 0000000..ca9e98c
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/rpc/README.md
@@ -0,0 +1,32 @@
+Spark Client RPC
+================
+
+Basic flow of events:
+
+- Client side creates an RPC server
+- Client side spawns RemoteDriver, which manages the SparkContext, and provides a secret
+- Client side sets up a timer to wait for RemoteDriver to connect back
+- RemoteDriver connects back to client, SASL handshake ensues
+- Connection is established and now there's a session between the client and the driver.
+
+Features of the RPC layer:
+
+- All messages serialized via Kryo
+- All messages are replied to. It's either an empty "ack" or an actual response - that depends
+ on the message.
+- RPC send API is asynchronous - callers get a future that can be used to wait for the message.
+- Currently, no connection retry. If connection goes down, both sides tear down the session.
+
+Notes:
+
+- Because serialization is using Kryo, types need explicit empty constructors or things will
+ fail to deserialize. This can be seen in the way exceptions are propagated - the throwing
+ side sends just a string stack trace to the remote, because certain fields on exceptions
+ don't have empty constructors.
+- The above is especially important because at the moment there's no way to register custom
+ serializers in the RPC library.
+
+Future work:
+
+- Random initial RPC id + id wrapping.
+- SSL / security in general.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/rpc/Rpc.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/rpc/Rpc.java b/rsc/src/main/java/org/apache/livy/rsc/rpc/Rpc.java
new file mode 100644
index 0000000..4d78a6a
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/rpc/Rpc.java
@@ -0,0 +1,460 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.rpc;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.sasl.RealmCallback;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslClient;
+import javax.security.sasl.SaslException;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.embedded.EmbeddedChannel;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
+import io.netty.util.concurrent.EventExecutorGroup;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+import io.netty.util.concurrent.ImmediateEventExecutor;
+import io.netty.util.concurrent.Promise;
+import io.netty.util.concurrent.ScheduledFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.rsc.RSCConf;
+import org.apache.livy.rsc.Utils;
+import static org.apache.livy.rsc.RSCConf.Entry.*;
+
+/**
+ * Encapsulates the RPC functionality. Provides higher-level methods to talk to the remote
+ * endpoint.
+ */
+public class Rpc implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Rpc.class);
+
+ static final String SASL_REALM = "rsc";
+ static final String SASL_USER = "rsc";
+ static final String SASL_PROTOCOL = "rsc";
+ static final String SASL_AUTH_CONF = "auth-conf";
+
+ /**
+ * Creates an RPC client for a server running on the given remote host and port.
+ *
+ * @param config RPC configuration data.
+ * @param eloop Event loop for managing the connection.
+ * @param host Host name or IP address to connect to.
+ * @param port Port where server is listening.
+ * @param clientId The client ID that identifies the connection.
+ * @param secret Secret for authenticating the client with the server.
+ * @param dispatcher Dispatcher used to handle RPC calls.
+ * @return A future that can be used to monitor the creation of the RPC object.
+ */
+ public static Promise<Rpc> createClient(
+ final RSCConf config,
+ final EventLoopGroup eloop,
+ String host,
+ int port,
+ final String clientId,
+ final String secret,
+ final RpcDispatcher dispatcher) throws Exception {
+ int connectTimeoutMs = (int) config.getTimeAsMs(RPC_CLIENT_CONNECT_TIMEOUT);
+
+ final ChannelFuture cf = new Bootstrap()
+ .group(eloop)
+ .handler(new ChannelInboundHandlerAdapter() { })
+ .channel(NioSocketChannel.class)
+ .option(ChannelOption.SO_KEEPALIVE, true)
+ .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, connectTimeoutMs)
+ .connect(host, port);
+
+ final Promise<Rpc> promise = eloop.next().newPromise();
+ final AtomicReference<Rpc> rpc = new AtomicReference<Rpc>();
+
+ // Set up a timeout to undo everything.
+ final Runnable timeoutTask = new Runnable() {
+ @Override
+ public void run() {
+ promise.setFailure(new TimeoutException("Timed out waiting for RPC server connection."));
+ }
+ };
+ final ScheduledFuture<?> timeoutFuture = eloop.schedule(timeoutTask,
+ config.getTimeAsMs(RPC_CLIENT_HANDSHAKE_TIMEOUT), TimeUnit.MILLISECONDS);
+
+ // The channel listener instantiates the Rpc instance when the connection is established,
+ // and initiates the SASL handshake.
+ cf.addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture cf) throws Exception {
+ if (cf.isSuccess()) {
+ SaslClientHandler saslHandler = new SaslClientHandler(config, clientId, promise,
+ timeoutFuture, secret, dispatcher);
+ Rpc rpc = createRpc(config, saslHandler, (SocketChannel) cf.channel(), eloop);
+ saslHandler.rpc = rpc;
+ saslHandler.sendHello(cf.channel());
+ } else {
+ promise.setFailure(cf.cause());
+ }
+ }
+ });
+
+ // Handle cancellation of the promise.
+ promise.addListener(new GenericFutureListener<Promise<Rpc>>() {
+ @Override
+ public void operationComplete(Promise<Rpc> p) {
+ if (p.isCancelled()) {
+ cf.cancel(true);
+ }
+ }
+ });
+
+ return promise;
+ }
+
+ static Rpc createServer(SaslHandler saslHandler, RSCConf config, SocketChannel channel,
+ EventExecutorGroup egroup) throws IOException {
+ return createRpc(config, saslHandler, channel, egroup);
+ }
+
+ private static Rpc createRpc(RSCConf config,
+ SaslHandler saslHandler,
+ SocketChannel client,
+ EventExecutorGroup egroup)
+ throws IOException {
+ LogLevel logLevel = LogLevel.TRACE;
+ String logLevelStr = config.get(RPC_CHANNEL_LOG_LEVEL);
+ if (logLevelStr != null) {
+ try {
+ logLevel = LogLevel.valueOf(logLevelStr);
+ } catch (Exception e) {
+ LOG.warn("Invalid log level {}, reverting to default.", logLevelStr);
+ }
+ }
+
+ boolean logEnabled = false;
+ switch (logLevel) {
+ case DEBUG:
+ logEnabled = LOG.isDebugEnabled();
+ break;
+ case ERROR:
+ logEnabled = LOG.isErrorEnabled();
+ break;
+ case INFO:
+ logEnabled = LOG.isInfoEnabled();
+ break;
+ case TRACE:
+ logEnabled = LOG.isTraceEnabled();
+ break;
+ case WARN:
+ logEnabled = LOG.isWarnEnabled();
+ break;
+ }
+
+ if (logEnabled) {
+ client.pipeline().addLast("logger", new LoggingHandler(Rpc.class, logLevel));
+ }
+
+ KryoMessageCodec kryo = new KryoMessageCodec(config.getInt(RPC_MAX_MESSAGE_SIZE),
+ MessageHeader.class, NullMessage.class, SaslMessage.class);
+ saslHandler.setKryoMessageCodec(kryo);
+ client.pipeline()
+ .addLast("codec", kryo)
+ .addLast("sasl", saslHandler);
+ return new Rpc(config, client, egroup);
+ }
+
+ static Rpc createEmbedded(RpcDispatcher dispatcher) {
+ EmbeddedChannel c = new EmbeddedChannel(
+ new LoggingHandler(Rpc.class),
+ new KryoMessageCodec(0, MessageHeader.class, NullMessage.class),
+ dispatcher);
+ Rpc rpc = new Rpc(new RSCConf(null), c, ImmediateEventExecutor.INSTANCE);
+ rpc.dispatcher = dispatcher;
+ return rpc;
+ }
+
+ private final RSCConf config;
+ private final AtomicBoolean rpcClosed;
+ private final AtomicLong rpcId;
+ private final Channel channel;
+ private final EventExecutorGroup egroup;
+ private volatile RpcDispatcher dispatcher;
+
+ private Rpc(RSCConf config, Channel channel, EventExecutorGroup egroup) {
+ Utils.checkArgument(channel != null);
+ Utils.checkArgument(egroup != null);
+ this.config = config;
+ this.channel = channel;
+ this.dispatcher = null;
+ this.egroup = egroup;
+ this.rpcClosed = new AtomicBoolean();
+ this.rpcId = new AtomicLong();
+
+ // Note: this does not work for embedded channels.
+ channel.pipeline().addLast("monitor", new ChannelInboundHandlerAdapter() {
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+ close();
+ super.channelInactive(ctx);
+ }
+ });
+ }
+
+ /**
+ * Send an RPC call to the remote endpoint and returns a future that can be used to monitor the
+ * operation.
+ */
+ public Future<Void> call(Object msg) {
+ return call(msg, Void.class);
+ }
+
+ /**
+ * Send an RPC call to the remote endpoint and returns a future that can be used to monitor the
+ * operation.
+ *
+ * @param msg RPC call to send.
+ * @param retType Type of expected reply.
+ * @return A future used to monitor the operation.
+ */
+ public <T> Future<T> call(final Object msg, Class<T> retType) {
+ Utils.checkArgument(msg != null);
+ Utils.checkState(channel.isOpen(), "RPC channel is closed.");
+ try {
+ final long id = rpcId.getAndIncrement();
+ final Promise<T> promise = egroup.next().newPromise();
+ final ChannelFutureListener listener = new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture cf) {
+ if (!cf.isSuccess() && !promise.isDone()) {
+ LOG.warn("Failed to send RPC, closing connection.", cf.cause());
+ promise.setFailure(cf.cause());
+ dispatcher.discardRpc(id);
+ close();
+ }
+ }
+ };
+
+ dispatcher.registerRpc(id, promise, msg.getClass().getName());
+ channel.eventLoop().submit(new Runnable() {
+ @Override
+ public void run() {
+ channel.write(new MessageHeader(id, Rpc.MessageType.CALL)).addListener(listener);
+ channel.writeAndFlush(msg).addListener(listener);
+ }
+ });
+
+ return promise;
+ } catch (Exception e) {
+ throw Utils.propagate(e);
+ }
+ }
+
+ public Channel getChannel() {
+ return channel;
+ }
+
+ void setDispatcher(RpcDispatcher dispatcher) {
+ Utils.checkNotNull(dispatcher);
+ Utils.checkState(this.dispatcher == null, "Dispatcher already set.");
+ this.dispatcher = dispatcher;
+ channel.pipeline().addLast("dispatcher", dispatcher);
+ }
+
+ @Override
+ public void close() {
+ if (!rpcClosed.compareAndSet(false, true)) {
+ return;
+ }
+ try {
+ channel.close().sync();
+ } catch (InterruptedException ie) {
+ Thread.interrupted();
+ }
+ }
+
+ static enum MessageType {
+ CALL,
+ REPLY,
+ ERROR;
+ }
+
+ static class MessageHeader {
+ final long id;
+ final MessageType type;
+
+ MessageHeader() {
+ this(-1, null);
+ }
+
+ MessageHeader(long id, MessageType type) {
+ this.id = id;
+ this.type = type;
+ }
+
+ }
+
+ static class NullMessage {
+
+ }
+
+ static class SaslMessage {
+ final String clientId;
+ final byte[] payload;
+
+ SaslMessage() {
+ this(null, null);
+ }
+
+ SaslMessage(byte[] payload) {
+ this(null, payload);
+ }
+
+ SaslMessage(String clientId, byte[] payload) {
+ this.clientId = clientId;
+ this.payload = payload;
+ }
+
+ }
+
+ private static class SaslClientHandler extends SaslHandler implements CallbackHandler {
+
+ private final SaslClient client;
+ private final String clientId;
+ private final String secret;
+ private final RpcDispatcher dispatcher;
+ private Promise<Rpc> promise;
+ private ScheduledFuture<?> timeout;
+
+ // Can't be set in constructor due to circular dependency.
+ private Rpc rpc;
+
+ SaslClientHandler(
+ RSCConf config,
+ String clientId,
+ Promise<Rpc> promise,
+ ScheduledFuture<?> timeout,
+ String secret,
+ RpcDispatcher dispatcher)
+ throws IOException {
+ super(config);
+ this.clientId = clientId;
+ this.promise = promise;
+ this.timeout = timeout;
+ this.secret = secret;
+ this.dispatcher = dispatcher;
+ this.client = Sasl.createSaslClient(new String[] { config.get(SASL_MECHANISMS) },
+ null, SASL_PROTOCOL, SASL_REALM, config.getSaslOptions(), this);
+ }
+
+ @Override
+ protected boolean isComplete() {
+ return client.isComplete();
+ }
+
+ @Override
+ protected String getNegotiatedProperty(String name) {
+ return (String) client.getNegotiatedProperty(name);
+ }
+
+ @Override
+ protected SaslMessage update(SaslMessage challenge) throws IOException {
+ byte[] response = client.evaluateChallenge(challenge.payload);
+ return response != null ? new SaslMessage(response) : null;
+ }
+
+ @Override
+ public byte[] wrap(byte[] data, int offset, int len) throws IOException {
+ return client.wrap(data, offset, len);
+ }
+
+ @Override
+ public byte[] unwrap(byte[] data, int offset, int len) throws IOException {
+ return client.unwrap(data, offset, len);
+ }
+
+ @Override
+ public void dispose() throws IOException {
+ if (!client.isComplete()) {
+ onError(new SaslException("Client closed before SASL negotiation finished."));
+ }
+ client.dispose();
+ }
+
+ @Override
+ protected void onComplete() throws Exception {
+ timeout.cancel(true);
+ rpc.setDispatcher(dispatcher);
+ promise.setSuccess(rpc);
+ timeout = null;
+ promise = null;
+ }
+
+ @Override
+ protected void onError(Throwable error) {
+ timeout.cancel(true);
+ if (!promise.isDone()) {
+ promise.setFailure(error);
+ }
+ }
+
+ @Override
+ public void handle(Callback[] callbacks) {
+ for (Callback cb : callbacks) {
+ if (cb instanceof NameCallback) {
+ ((NameCallback)cb).setName(clientId);
+ } else if (cb instanceof PasswordCallback) {
+ ((PasswordCallback)cb).setPassword(secret.toCharArray());
+ } else if (cb instanceof RealmCallback) {
+ RealmCallback rb = (RealmCallback) cb;
+ rb.setText(rb.getDefaultText());
+ }
+ }
+ }
+
+ void sendHello(Channel c) throws Exception {
+ byte[] hello = client.hasInitialResponse() ?
+ client.evaluateChallenge(new byte[0]) : new byte[0];
+ c.writeAndFlush(new SaslMessage(clientId, hello)).sync();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcDispatcher.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcDispatcher.java b/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcDispatcher.java
new file mode 100644
index 0000000..c857a74
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcDispatcher.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.rpc;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.util.concurrent.Promise;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.rsc.Utils;
+
+/**
+ * An implementation of ChannelInboundHandler that dispatches incoming messages to an instance
+ * method based on the method signature.
+ * <p/>
+ * A handler's signature must be of the form:
+ * <p/>
+ * <blockquote><tt>protected void handle(ChannelHandlerContext, MessageType)</tt></blockquote>
+ * <p/>
+ * Where "MessageType" must match exactly the type of the message to handle. Polymorphism is not
+ * supported. Handlers can return a value, which becomes the RPC reply; if a null is returned, then
+ * a reply is still sent, with an empty payload.
+ */
+public abstract class RpcDispatcher extends SimpleChannelInboundHandler<Object> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RpcDispatcher.class);
+
+ private final Map<Class<?>, Method> handlers = new ConcurrentHashMap<>();
+ private final Collection<OutstandingRpc> rpcs = new ConcurrentLinkedQueue<OutstandingRpc>();
+
+ private volatile Rpc.MessageHeader lastHeader;
+
+ /** Override this to add a name to the dispatcher, for debugging purposes. */
+ protected String name() {
+ return getClass().getSimpleName();
+ }
+
+ @Override
+ protected final void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception {
+ if (lastHeader == null) {
+ if (!(msg instanceof Rpc.MessageHeader)) {
+ LOG.warn("[{}] Expected RPC header, got {} instead.", name(),
+ msg != null ? msg.getClass().getName() : null);
+ throw new IllegalArgumentException();
+ }
+ lastHeader = (Rpc.MessageHeader) msg;
+ } else {
+ LOG.debug("[{}] Received RPC message: type={} id={} payload={}", name(),
+ lastHeader.type, lastHeader.id, msg != null ? msg.getClass().getName() : null);
+ try {
+ switch (lastHeader.type) {
+ case CALL:
+ handleCall(ctx, msg);
+ break;
+ case REPLY:
+ handleReply(ctx, msg, findRpc(lastHeader.id));
+ break;
+ case ERROR:
+ handleError(ctx, msg, findRpc(lastHeader.id));
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown RPC message type: " + lastHeader.type);
+ }
+ } finally {
+ lastHeader = null;
+ }
+ }
+ }
+
+ private OutstandingRpc findRpc(long id) {
+ for (Iterator<OutstandingRpc> it = rpcs.iterator(); it.hasNext();) {
+ OutstandingRpc rpc = it.next();
+ if (rpc.id == id) {
+ it.remove();
+ return rpc;
+ }
+ }
+ throw new IllegalArgumentException(String.format(
+ "Received RPC reply for unknown RPC (%d).", id));
+ }
+
+ private void handleCall(ChannelHandlerContext ctx, Object msg) throws Exception {
+ Method handler = handlers.get(msg.getClass());
+ if (handler == null) {
+ // Try both getDeclaredMethod() and getMethod() so that we try both private methods
+ // of the class, and public methods of parent classes.
+ try {
+ handler = getClass().getDeclaredMethod("handle", ChannelHandlerContext.class,
+ msg.getClass());
+ } catch (NoSuchMethodException e) {
+ try {
+ handler = getClass().getMethod("handle", ChannelHandlerContext.class,
+ msg.getClass());
+ } catch (NoSuchMethodException e2) {
+ LOG.warn(String.format("[%s] Failed to find handler for msg '%s'.", name(),
+ msg.getClass().getName()));
+ writeMessage(ctx, Rpc.MessageType.ERROR, Utils.stackTraceAsString(e.getCause()));
+ return;
+ }
+ }
+ handler.setAccessible(true);
+ handlers.put(msg.getClass(), handler);
+ }
+
+ try {
+ Object payload = handler.invoke(this, ctx, msg);
+ if (payload == null) {
+ payload = new Rpc.NullMessage();
+ }
+ writeMessage(ctx, Rpc.MessageType.REPLY, payload);
+ } catch (InvocationTargetException ite) {
+ LOG.debug(String.format("[%s] Error in RPC handler.", name()), ite.getCause());
+ writeMessage(ctx, Rpc.MessageType.ERROR, Utils.stackTraceAsString(ite.getCause()));
+ }
+ }
+
+ private void writeMessage(ChannelHandlerContext ctx, Rpc.MessageType replyType, Object payload) {
+ ctx.channel().write(new Rpc.MessageHeader(lastHeader.id, replyType));
+ ctx.channel().writeAndFlush(payload);
+ }
+
+ private void handleReply(ChannelHandlerContext ctx, Object msg, OutstandingRpc rpc)
+ throws Exception {
+ rpc.future.setSuccess(msg instanceof Rpc.NullMessage ? null : msg);
+ }
+
+ private void handleError(ChannelHandlerContext ctx, Object msg, OutstandingRpc rpc)
+ throws Exception {
+ if (msg instanceof String) {
+ LOG.warn("Received error message:{}.", msg);
+ rpc.future.setFailure(new RpcException((String) msg));
+ } else {
+ String error = String.format("Received error with unexpected payload (%s).",
+ msg != null ? msg.getClass().getName() : null);
+ LOG.warn(String.format("[%s] %s", name(), error));
+ rpc.future.setFailure(new IllegalArgumentException(error));
+ ctx.close();
+ }
+ }
+
+ @Override
+ public final void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("[%s] Caught exception in channel pipeline.", name()), cause);
+ } else {
+ LOG.info("[{}] Closing channel due to exception in pipeline ({}).", name(),
+ cause.getMessage());
+ }
+
+ if (lastHeader != null) {
+ // There's an RPC waiting for a reply. Exception was most probably caught while processing
+ // the RPC, so send an error.
+ ctx.channel().write(new Rpc.MessageHeader(lastHeader.id, Rpc.MessageType.ERROR));
+ ctx.channel().writeAndFlush(Utils.stackTraceAsString(cause));
+ lastHeader = null;
+ }
+
+ ctx.close();
+ }
+
+ @Override
+ public final void channelInactive(ChannelHandlerContext ctx) throws Exception {
+ if (rpcs.size() > 0) {
+ LOG.warn("[{}] Closing RPC channel with {} outstanding RPCs.", name(), rpcs.size());
+ for (OutstandingRpc rpc : rpcs) {
+ rpc.future.cancel(true);
+ }
+ } else {
+ LOG.debug("Channel {} became inactive.", ctx.channel());
+ }
+ super.channelInactive(ctx);
+ }
+
+ void registerRpc(long id, Promise<?> promise, String type) {
+ LOG.debug("[{}] Registered outstanding rpc {} ({}).", name(), id, type);
+ rpcs.add(new OutstandingRpc(id, promise));
+ }
+
+ void discardRpc(long id) {
+ LOG.debug("[{}] Discarding failed RPC {}.", name(), id);
+ findRpc(id);
+ }
+
+ private static class OutstandingRpc {
+ final long id;
+ final Promise<Object> future;
+
+ @SuppressWarnings("unchecked")
+ OutstandingRpc(long id, Promise<?> future) {
+ this.id = id;
+ this.future = (Promise<Object>) future;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcException.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcException.java b/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcException.java
new file mode 100644
index 0000000..2c2d39d
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcException.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.rpc;
+
+public class RpcException extends RuntimeException {
+
+ RpcException(String remoteStackTrace) {
+ super(remoteStackTrace);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcServer.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcServer.java b/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcServer.java
new file mode 100644
index 0000000..bdd3de6
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/rpc/RpcServer.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.rpc;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.BindException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.SocketException;
+import java.security.SecureRandom;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.sasl.AuthorizeCallback;
+import javax.security.sasl.RealmCallback;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslException;
+import javax.security.sasl.SaslServer;
+
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.util.concurrent.ScheduledFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.rsc.RSCConf;
+import org.apache.livy.rsc.Utils;
+import static org.apache.livy.rsc.RSCConf.Entry.*;
+
+/**
+ * An RPC server. The server matches remote clients based on a secret that is generated on
+ * the server - the secret needs to be given to the client through some other mechanism for
+ * this to work.
+ */
+public class RpcServer implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RpcServer.class);
+ private static final SecureRandom RND = new SecureRandom();
+
+ private final String address;
+ private Channel channel;
+ private final EventLoopGroup group;
+ private final int port;
+ private final ConcurrentMap<String, ClientInfo> pendingClients;
+ private final RSCConf config;
+ private final String portRange;
+ private static enum PortRangeSchema{START_PORT, END_PORT, MAX};
+ private final String PORT_DELIMITER = "~";
+ /**
+ * Creating RPC Server
+ * @param lconf
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public RpcServer(RSCConf lconf) throws IOException, InterruptedException {
+ this.config = lconf;
+ this.portRange = config.get(LAUNCHER_PORT_RANGE);
+ this.group = new NioEventLoopGroup(
+ this.config.getInt(RPC_MAX_THREADS),
+ Utils.newDaemonThreadFactory("RPC-Handler-%d"));
+ int [] portData = getPortNumberAndRange();
+ int startingPortNumber = portData[PortRangeSchema.START_PORT.ordinal()];
+ int endPort = portData[PortRangeSchema.END_PORT.ordinal()];
+ boolean isContected = false;
+ for(int tries = startingPortNumber ; tries<=endPort ; tries++){
+ try {
+ this.channel = getChannel(tries);
+ isContected = true;
+ break;
+ } catch(SocketException e){
+ LOG.debug("RPC not able to connect port " + tries + " " + e.getMessage());
+ }
+ }
+ if(!isContected) {
+ throw new IOException("Unable to connect to provided ports " + this.portRange);
+ }
+ this.port = ((InetSocketAddress) channel.localAddress()).getPort();
+ this.pendingClients = new ConcurrentHashMap<>();
+ LOG.info("Connected to the port " + this.port);
+ String address = config.get(RPC_SERVER_ADDRESS);
+ if (address == null) {
+ address = config.findLocalAddress();
+ }
+ this.address = address;
+ }
+
+ /**
+ * Get Port Numbers
+ */
+ private int[] getPortNumberAndRange() throws ArrayIndexOutOfBoundsException,
+ NumberFormatException {
+ String[] split = this.portRange.split(PORT_DELIMITER);
+ int [] portRange = new int [PortRangeSchema.MAX.ordinal()];
+ try {
+ portRange[PortRangeSchema.START_PORT.ordinal()] =
+ Integer.parseInt(split[PortRangeSchema.START_PORT.ordinal()]);
+ portRange[PortRangeSchema.END_PORT.ordinal()] =
+ Integer.parseInt(split[PortRangeSchema.END_PORT.ordinal()]);
+ } catch(ArrayIndexOutOfBoundsException e) {
+ LOG.error("Port Range format is not correct " + this.portRange);
+ throw e;
+ } catch(NumberFormatException e) {
+ LOG.error("Port are not in numeric format " + this.portRange);
+ throw e;
+ }
+ return portRange;
+ }
+ /**
+ * @throws InterruptedException
+ **/
+ private Channel getChannel(int portNumber) throws BindException, InterruptedException {
+ Channel channel = new ServerBootstrap()
+ .group(group)
+ .channel(NioServerSocketChannel.class)
+ .childHandler(new ChannelInitializer<SocketChannel>() {
+ @Override
+ public void initChannel(SocketChannel ch) throws Exception {
+ SaslServerHandler saslHandler = new SaslServerHandler(config);
+ final Rpc newRpc = Rpc.createServer(saslHandler, config, ch, group);
+ saslHandler.rpc = newRpc;
+
+ Runnable cancelTask = new Runnable() {
+ @Override
+ public void run() {
+ LOG.warn("Timed out waiting for hello from client.");
+ newRpc.close();
+ }
+ };
+ saslHandler.cancelTask = group.schedule(cancelTask,
+ config.getTimeAsMs(RPC_CLIENT_HANDSHAKE_TIMEOUT),
+ TimeUnit.MILLISECONDS);
+ }
+ })
+ .option(ChannelOption.SO_BACKLOG, 1)
+ .option(ChannelOption.SO_REUSEADDR, true)
+ .childOption(ChannelOption.SO_KEEPALIVE, true)
+ .bind(portNumber)
+ .sync()
+ .channel();
+ return channel;
+ }
+ /**
+ * Tells the RPC server to expect connections from clients.
+ *
+ * @param clientId An identifier for the client. Must be unique.
+ * @param secret The secret the client will send to the server to identify itself.
+ * @param callback The callback for when a new client successfully connects with the given
+ * credentials.
+ */
+ public void registerClient(String clientId, String secret, ClientCallback callback) {
+ final ClientInfo client = new ClientInfo(clientId, secret, callback);
+ if (pendingClients.putIfAbsent(clientId, client) != null) {
+ throw new IllegalStateException(
+ String.format("Client '%s' already registered.", clientId));
+ }
+ }
+
+ /**
+ * Stop waiting for connections for a given client ID.
+ *
+ * @param clientId The client ID to forget.
+ */
+ public void unregisterClient(String clientId) {
+ pendingClients.remove(clientId);
+ }
+
+ /**
+ * Creates a secret for identifying a client connection.
+ */
+ public String createSecret() {
+ byte[] secret = new byte[config.getInt(RPC_SECRET_RANDOM_BITS) / 8];
+ RND.nextBytes(secret);
+
+ StringBuilder sb = new StringBuilder();
+ for (byte b : secret) {
+ if (b < 10) {
+ sb.append("0");
+ }
+ sb.append(Integer.toHexString(b));
+ }
+ return sb.toString();
+ }
+
+ public String getAddress() {
+ return address;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public EventLoopGroup getEventLoopGroup() {
+ return group;
+ }
+
+ @Override
+ public void close() {
+ try {
+ channel.close();
+ pendingClients.clear();
+ } finally {
+ group.shutdownGracefully();
+ }
+ }
+
+ /**
+ * A callback that can be registered to be notified when new clients are created and
+ * successfully authenticate against the server.
+ */
+ public interface ClientCallback {
+
+ /**
+ * Called when a new client successfully connects.
+ *
+ * @param client The RPC instance for the new client.
+ * @return The RpcDispatcher to be used for the client.
+ */
+ RpcDispatcher onNewClient(Rpc client);
+
+
+ /**
+ * Called when a new client successfully completed SASL authentication.
+ *
+ * @param client The RPC instance for the new client.
+ */
+ void onSaslComplete(Rpc client);
+ }
+
+ private class SaslServerHandler extends SaslHandler implements CallbackHandler {
+
+ private final SaslServer server;
+ private Rpc rpc;
+ private ScheduledFuture<?> cancelTask;
+ private String clientId;
+ private ClientInfo client;
+
+ SaslServerHandler(RSCConf config) throws IOException {
+ super(config);
+ this.server = Sasl.createSaslServer(config.get(SASL_MECHANISMS), Rpc.SASL_PROTOCOL,
+ Rpc.SASL_REALM, config.getSaslOptions(), this);
+ }
+
+ @Override
+ protected boolean isComplete() {
+ return server.isComplete();
+ }
+
+ @Override
+ protected String getNegotiatedProperty(String name) {
+ return (String) server.getNegotiatedProperty(name);
+ }
+
+ @Override
+ protected Rpc.SaslMessage update(Rpc.SaslMessage challenge) throws IOException {
+ if (clientId == null) {
+ Utils.checkArgument(challenge.clientId != null,
+ "Missing client ID in SASL handshake.");
+ clientId = challenge.clientId;
+ client = pendingClients.get(clientId);
+ Utils.checkArgument(client != null,
+ "Unexpected client ID '%s' in SASL handshake.", clientId);
+ }
+
+ return new Rpc.SaslMessage(server.evaluateResponse(challenge.payload));
+ }
+
+ @Override
+ public byte[] wrap(byte[] data, int offset, int len) throws IOException {
+ return server.wrap(data, offset, len);
+ }
+
+ @Override
+ public byte[] unwrap(byte[] data, int offset, int len) throws IOException {
+ return server.unwrap(data, offset, len);
+ }
+
+ @Override
+ public void dispose() throws IOException {
+ if (!server.isComplete()) {
+ onError(new SaslException("Server closed before SASL negotiation finished."));
+ }
+ server.dispose();
+ }
+
+ @Override
+ protected void onComplete() throws Exception {
+ cancelTask.cancel(true);
+
+ RpcDispatcher dispatcher = null;
+ try {
+ dispatcher = client.callback.onNewClient(rpc);
+ } catch (Exception e) {
+ LOG.warn("Client callback threw an exception.", e);
+ }
+
+ if (dispatcher != null) {
+ rpc.setDispatcher(dispatcher);
+ }
+
+ client.callback.onSaslComplete(rpc);
+ }
+
+ @Override
+ protected void onError(Throwable error) {
+ cancelTask.cancel(true);
+ }
+
+ @Override
+ public void handle(Callback[] callbacks) {
+ Utils.checkState(client != null, "Handshake not initialized yet.");
+ for (Callback cb : callbacks) {
+ if (cb instanceof NameCallback) {
+ ((NameCallback)cb).setName(clientId);
+ } else if (cb instanceof PasswordCallback) {
+ ((PasswordCallback)cb).setPassword(client.secret.toCharArray());
+ } else if (cb instanceof AuthorizeCallback) {
+ ((AuthorizeCallback) cb).setAuthorized(true);
+ } else if (cb instanceof RealmCallback) {
+ RealmCallback rb = (RealmCallback) cb;
+ rb.setText(rb.getDefaultText());
+ }
+ }
+ }
+
+ }
+
+ private static class ClientInfo {
+
+ final String id;
+ final String secret;
+ final ClientCallback callback;
+
+ private ClientInfo(String id, String secret, ClientCallback callback) {
+ this.id = id;
+ this.secret = secret;
+ this.callback = callback;
+ }
+
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/rpc/SaslHandler.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/rpc/SaslHandler.java b/rsc/src/main/java/org/apache/livy/rsc/rpc/SaslHandler.java
new file mode 100644
index 0000000..ec18a70
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/rpc/SaslHandler.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.rpc;
+
+import java.io.IOException;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslException;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.rsc.RSCConf;
+
+/**
+ * Abstract SASL handler. Abstracts the auth protocol handling and encryption, if it's enabled.
+ * Needs subclasses to provide access to the actual underlying SASL implementation (client or
+ * server).
+ */
+abstract class SaslHandler extends SimpleChannelInboundHandler<Rpc.SaslMessage>
+ implements KryoMessageCodec.EncryptionHandler {
+
+ // LOG is not static to make debugging easier (being able to identify which sub-class
+ // generated the log message).
+ private final Logger LOG;
+ private final boolean requiresEncryption;
+ private KryoMessageCodec kryo;
+ private boolean hasAuthResponse = false;
+
+ protected SaslHandler(RSCConf config) {
+ this.requiresEncryption = Rpc.SASL_AUTH_CONF.equals(config.get(RSCConf.Entry.SASL_QOP));
+ this.LOG = LoggerFactory.getLogger(getClass());
+ }
+
+ // Use a separate method to make it easier to create a SaslHandler without having to
+ // plumb the KryoMessageCodec instance through the constructors.
+ void setKryoMessageCodec(KryoMessageCodec kryo) {
+ this.kryo = kryo;
+ }
+
+ @Override
+ protected final void channelRead0(ChannelHandlerContext ctx, Rpc.SaslMessage msg)
+ throws Exception {
+ LOG.debug("Handling SASL challenge message...");
+ Rpc.SaslMessage response = update(msg);
+ if (response != null) {
+ LOG.debug("Sending SASL challenge response...");
+ hasAuthResponse = true;
+ ctx.channel().writeAndFlush(response).sync();
+ }
+
+ if (!isComplete()) {
+ return;
+ }
+
+ // If negotiation is complete, remove this handler from the pipeline, and register it with
+ // the Kryo instance to handle encryption if needed.
+ ctx.channel().pipeline().remove(this);
+ String qop = getNegotiatedProperty(Sasl.QOP);
+ LOG.debug("SASL negotiation finished with QOP {}.", qop);
+ if (Rpc.SASL_AUTH_CONF.equals(qop)) {
+ LOG.info("SASL confidentiality enabled.");
+ kryo.setEncryptionHandler(this);
+ } else {
+ if (requiresEncryption) {
+ throw new SaslException("Encryption required, but SASL negotiation did not set it up.");
+ }
+ dispose();
+ }
+
+ onComplete();
+ }
+
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+ dispose();
+ super.channelInactive(ctx);
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+ if (!isComplete()) {
+ LOG.info("Exception in SASL negotiation.", cause);
+ onError(cause);
+ ctx.close();
+ }
+ ctx.fireExceptionCaught(cause);
+ }
+
+ protected abstract boolean isComplete();
+
+ protected abstract String getNegotiatedProperty(String name);
+
+ protected abstract Rpc.SaslMessage update(Rpc.SaslMessage challenge) throws IOException;
+
+ protected abstract void onComplete() throws Exception;
+
+ protected abstract void onError(Throwable t);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/resources/META-INF/services/com.cloudera.livy.LivyClientFactory
----------------------------------------------------------------------
diff --git a/rsc/src/main/resources/META-INF/services/com.cloudera.livy.LivyClientFactory b/rsc/src/main/resources/META-INF/services/com.cloudera.livy.LivyClientFactory
deleted file mode 100644
index bd50e71..0000000
--- a/rsc/src/main/resources/META-INF/services/com.cloudera.livy.LivyClientFactory
+++ /dev/null
@@ -1 +0,0 @@
-com.cloudera.livy.rsc.RSCClientFactory
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/resources/META-INF/services/org.apache.livy.LivyClientFactory
----------------------------------------------------------------------
diff --git a/rsc/src/main/resources/META-INF/services/org.apache.livy.LivyClientFactory b/rsc/src/main/resources/META-INF/services/org.apache.livy.LivyClientFactory
new file mode 100644
index 0000000..7226429
--- /dev/null
+++ b/rsc/src/main/resources/META-INF/services/org.apache.livy.LivyClientFactory
@@ -0,0 +1 @@
+org.apache.livy.rsc.RSCClientFactory
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/test/java/com/cloudera/livy/rsc/TestJobHandle.java
----------------------------------------------------------------------
diff --git a/rsc/src/test/java/com/cloudera/livy/rsc/TestJobHandle.java b/rsc/src/test/java/com/cloudera/livy/rsc/TestJobHandle.java
deleted file mode 100644
index a2453eb..0000000
--- a/rsc/src/test/java/com/cloudera/livy/rsc/TestJobHandle.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import io.netty.util.concurrent.Promise;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InOrder;
-import org.mockito.Mock;
-import org.mockito.runners.MockitoJUnitRunner;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
-
-import com.cloudera.livy.JobHandle;
-
-@RunWith(MockitoJUnitRunner.class)
-public class TestJobHandle {
-
- @Mock private RSCClient client;
- @Mock private Promise<Object> promise;
- @Mock private JobHandle.Listener<Object> listener;
- @Mock private JobHandle.Listener<Object> listener2;
-
- @Test
- public void testStateChanges() throws Exception {
- JobHandleImpl<Object> handle = new JobHandleImpl<Object>(client, promise, "job");
- handle.addListener(listener);
-
- assertTrue(handle.changeState(JobHandle.State.QUEUED));
- verify(listener).onJobQueued(handle);
-
- assertTrue(handle.changeState(JobHandle.State.STARTED));
- verify(listener).onJobStarted(handle);
-
- assertTrue(handle.changeState(JobHandle.State.CANCELLED));
- verify(listener).onJobCancelled(handle);
-
- assertFalse(handle.changeState(JobHandle.State.STARTED));
- assertFalse(handle.changeState(JobHandle.State.FAILED));
- assertFalse(handle.changeState(JobHandle.State.SUCCEEDED));
- }
-
- @Test
- public void testFailedJob() throws Exception {
- JobHandleImpl<Object> handle = new JobHandleImpl<Object>(client, promise, "job");
- handle.addListener(listener);
-
- Throwable cause = new Exception();
- when(promise.cause()).thenReturn(cause);
-
- assertTrue(handle.changeState(JobHandle.State.FAILED));
- verify(promise).cause();
- verify(listener).onJobFailed(handle, cause);
- }
-
- @Test
- public void testSucceededJob() throws Exception {
- JobHandleImpl<Object> handle = new JobHandleImpl<Object>(client, promise, "job");
- handle.addListener(listener);
-
- Object result = new Exception();
- when(promise.getNow()).thenReturn(result);
-
- assertTrue(handle.changeState(JobHandle.State.SUCCEEDED));
- verify(promise).getNow();
- verify(listener).onJobSucceeded(handle, result);
- }
-
- @Test
- public void testImmediateCallback() throws Exception {
- JobHandleImpl<Object> handle = new JobHandleImpl<Object>(client, promise, "job");
- assertTrue(handle.changeState(JobHandle.State.QUEUED));
- handle.addListener(listener);
- verify(listener).onJobQueued(handle);
-
- handle.changeState(JobHandle.State.STARTED);
- handle.changeState(JobHandle.State.CANCELLED);
-
- handle.addListener(listener2);
- verify(listener2).onJobCancelled(same(handle));
- }
-
-}
[19/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/main/scala/com/cloudera/livy/scalaapi/LivyScalaClient.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/main/scala/com/cloudera/livy/scalaapi/LivyScalaClient.scala b/scala-api/src/main/scala/com/cloudera/livy/scalaapi/LivyScalaClient.scala
deleted file mode 100644
index 3d26c7a..0000000
--- a/scala-api/src/main/scala/com/cloudera/livy/scalaapi/LivyScalaClient.scala
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.scalaapi
-
-import java.io.File
-import java.net.URI
-import java.util.concurrent.{Executors, Future => JFuture, ScheduledFuture, ThreadFactory, TimeUnit}
-
-import scala.concurrent._
-import scala.util.Try
-
-import com.cloudera.livy._
-
-/**
- * A client for submitting Spark-based jobs to a Livy backend.
- * @constructor Creates a Scala client.
- * @param livyJavaClient the Java client of Livy.
- */
-class LivyScalaClient(livyJavaClient: LivyClient) {
-
- private val executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory {
- override def newThread(r: Runnable): Thread = {
- val thread = new Thread(r, "LivyScalaClient-PollingContainer")
- thread.setDaemon(true)
- thread
- }
- })
-
- /**
- * Submits a job for asynchronous execution.
- *
- * @param fn The job to be executed. It is a function that takes in a ScalaJobContext and
- * returns the result of the execution of the job with that context.
- * @return A handle that can be used to monitor the job.
- */
- def submit[T](fn: ScalaJobContext => T): ScalaJobHandle[T] = {
- val job = new Job[T] {
- @throws(classOf[Exception])
- override def call(jobContext: JobContext): T = fn(new ScalaJobContext(jobContext))
- }
- new ScalaJobHandle(livyJavaClient.submit(job))
- }
-
- /**
- * Asks the remote context to run a job immediately.
- *
- * Normally, the remote context will queue jobs and execute them based on how many worker
- * threads have been configured. This method will run the submitted job in the same thread
- * processing the RPC message, so that queueing does not apply.
- *
- * It's recommended that this method only be used to run code that finishes quickly. This
- * avoids interfering with the normal operation of the context.
- *
- * @param fn The job to be executed. It is a function that takes in a ScalaJobContext and
- * returns the result of the execution of the job with that context.
- * @return A handle that can be used to monitor the job.
- */
- def run[T](fn: ScalaJobContext => T): Future[T] = {
- val job = new Job[T] {
- @throws(classOf[Exception])
- override def call(jobContext: JobContext): T = {
- val scalaJobContext = new ScalaJobContext(jobContext)
- fn(scalaJobContext)
- }
- }
- new PollingContainer(livyJavaClient.run(job)).poll()
- }
-
- /**
- * Stops the remote context.
- *
- * Any pending jobs will be cancelled, and the remote context will be torn down.
- *
- * @param shutdownContext Whether to shutdown the underlying Spark context. If false, the
- * context will keep running and it's still possible to send commands
- * to it, if the backend being used supports it.
- */
- def stop(shutdownContext: Boolean): Unit = {
- executor.shutdown()
- livyJavaClient.stop(shutdownContext)
- }
-
- /**
- * Upload a jar to be added to the Spark application classpath.
- *
- * @param jar The local file to be uploaded.
- * @return A future that can be used to monitor this operation.
- */
- def uploadJar(jar: File): Future[_] = new PollingContainer(livyJavaClient.uploadJar(jar)).poll()
-
- /**
- * Adds a jar file to the running remote context.
- *
- * Note that the URL should be reachable by the Spark driver process. If running the driver
- * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
- * on that node (and not on the client machine).
- *
- * If the provided URI has no scheme, it's considered to be relative to the default file system
- * configured in the Livy server.
- *
- * @param uri The location of the jar file.
- * @return A future that can be used to monitor the operation.
- */
- def addJar(uri: URI): Future[_] = new PollingContainer(livyJavaClient.addJar(uri)).poll()
-
- /**
- * Upload a file to be passed to the Spark application.
- *
- * @param file The local file to be uploaded.
- * @return A future that can be used to monitor this operation.
- */
- def uploadFile(file: File): Future[_] =
- new PollingContainer(livyJavaClient.uploadFile(file)).poll()
-
- /**
- * Adds a file to the running remote context.
- *
- * Note that the URL should be reachable by the Spark driver process. If running the driver
- * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
- * on that node (and not on the client machine).
- *
- * If the provided URI has no scheme, it's considered to be relative to the default file system
- * configured in the Livy server.
- *
- * @param uri The location of the file.
- * @return A future that can be used to monitor the operation.
- */
- def addFile(uri: URI): Future[_] = new PollingContainer(livyJavaClient.addFile(uri)).poll()
-
- private class PollingContainer[T] private[livy] (jFuture: JFuture[T]) extends Runnable {
-
- private val initialDelay = 1
- private val longDelay = 1
- private var scheduledFuture: ScheduledFuture[_] = _
- private val promise = Promise[T]
-
- def poll(): Future[T] = {
- scheduledFuture =
- executor.scheduleWithFixedDelay(this, initialDelay, longDelay, TimeUnit.SECONDS)
- promise.future
- }
-
- override def run(): Unit = {
- if (jFuture.isDone) {
- promise.complete(Try(getJavaFutureResult(jFuture)))
- scheduledFuture.cancel(false)
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/main/scala/com/cloudera/livy/scalaapi/ScalaJobContext.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/main/scala/com/cloudera/livy/scalaapi/ScalaJobContext.scala b/scala-api/src/main/scala/com/cloudera/livy/scalaapi/ScalaJobContext.scala
deleted file mode 100644
index 57a3ab4..0000000
--- a/scala-api/src/main/scala/com/cloudera/livy/scalaapi/ScalaJobContext.scala
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.scalaapi
-
-import java.io.File
-
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.streaming.StreamingContext
-
-import com.cloudera.livy.JobContext
-
-/**
- * Holds runtime information about the job execution context.
- *
- * @constructor Creates a ScalaJobContext.
- * @param context the Java JobContext of Livy.
- */
-class ScalaJobContext private[livy] (context: JobContext) {
-
- /** The shared SparkContext instance. */
- def sc: SparkContext = context.sc().sc
-
- /** The shared SQLContext instance. */
- def sqlctx: SQLContext = context.sqlctx()
-
- /** The shared HiveContext instance. */
- def hivectx: HiveContext = context.hivectx()
-
- /** Returns the StreamingContext which has already been created. */
- def streamingctx: StreamingContext = context.streamingctx().ssc
-
- def sparkSession[E]: E = context.sparkSession()
-
- /**
- * Creates the SparkStreaming context.
- *
- * @param batchDuration Time interval at which streaming data will be divided into batches,
- * in milliseconds.
- */
- def createStreamingContext(batchDuration: Long): Unit =
- context.createStreamingContext(batchDuration)
-
- /** Stops the SparkStreaming context. */
- def stopStreamingContext(): Unit = context.stopStreamingCtx()
-
- /**
- * Returns a local tmp dir specific to the context.
- */
- def localTmpDir: File = context.getLocalTmpDir
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/main/scala/com/cloudera/livy/scalaapi/ScalaJobHandle.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/main/scala/com/cloudera/livy/scalaapi/ScalaJobHandle.scala b/scala-api/src/main/scala/com/cloudera/livy/scalaapi/ScalaJobHandle.scala
deleted file mode 100644
index 7bd2edc..0000000
--- a/scala-api/src/main/scala/com/cloudera/livy/scalaapi/ScalaJobHandle.scala
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.scalaapi
-
-import scala.concurrent.{CanAwait, ExecutionContext, Future, TimeoutException}
-import scala.concurrent.duration.Duration
-import scala.util.Try
-
-import com.cloudera.livy.JobHandle
-import com.cloudera.livy.JobHandle.{Listener, State}
-
-/**
- * A handle to a submitted job. Allows for monitoring and controlling of the running remote job.
- *
- * @constructor Creates a ScalaJobHandle.
- * @param jobHandle the Java JobHandle of Livy.
- *
- * @define multipleCallbacks
- * Multiple callbacks may be registered; there is no guarantee that they will be
- * executed in a particular order.
- *
- * @define nonDeterministic
- * Note: using this method yields nondeterministic dataflow programs.
- *
- * @define callbackInContext
- * The provided callback always runs in the provided implicit
- *` ExecutionContext`, though there is no guarantee that the
- * `execute()` method on the `ExecutionContext` will be called once
- * per callback or that `execute()` will be called in the current
- * thread. That is, the implementation may run multiple callbacks
- * in a batch within a single `execute()` and it may run
- * `execute()` either immediately or asynchronously.
- */
-class ScalaJobHandle[T] private[livy] (jobHandle: JobHandle[T]) extends Future[T] {
-
- /**
- * Return the current state of the job.
- */
- def state: State = jobHandle.getState()
-
- /**
- * When the job is completed, either through an exception, or a value,
- * apply the provided function.
- *
- * If the job has already been completed,
- * this will either be applied immediately or be scheduled asynchronously.
- *
- * $multipleCallbacks
- * $callbackInContext
- */
- override def onComplete[U](func: (Try[T]) => U)(implicit executor: ExecutionContext): Unit = {
- jobHandle.addListener(new AbstractScalaJobHandleListener[T] {
- override def onJobSucceeded(job: JobHandle[T], result: T): Unit = {
- val onJobSucceededTask = new Runnable {
- override def run(): Unit = func(Try(result))
- }
- executor.execute(onJobSucceededTask)
- }
-
- override def onJobFailed(job: JobHandle[T], cause: Throwable): Unit = {
- val onJobFailedTask = new Runnable {
- override def run(): Unit = func(Try(getJavaFutureResult(job)))
- }
- executor.execute(onJobFailedTask)
- }
- })
- }
-
- /**
- * When this job is queued, apply the provided function.
- *
- * $multipleCallbacks
- * $callbackInContext
- */
- def onJobQueued[U](func: => Unit)(implicit executor: ExecutionContext): Unit = {
- jobHandle.addListener(new AbstractScalaJobHandleListener[T] {
- override def onJobQueued(job: JobHandle[T]): Unit = {
- val onJobQueuedTask = new Runnable {
- override def run(): Unit = func
- }
- executor.execute(onJobQueuedTask)
- }
- })
- }
-
- /**
- * When this job has started, apply the provided function.
- *
- * $multipleCallbacks
- * $callbackInContext
- */
- def onJobStarted[U](func: => Unit)(implicit executor: ExecutionContext): Unit = {
- jobHandle.addListener(new AbstractScalaJobHandleListener[T] {
- override def onJobStarted(job: JobHandle[T]): Unit = {
- val onJobStartedTask = new Runnable {
- override def run(): Unit = func
- }
- executor.execute(onJobStartedTask)
- }
- })
- }
-
- /**
- * When this job is cancelled, apply the provided function.
- *
- * $multipleCallbacks
- * $callbackInContext
- */
- def onJobCancelled[U](func: Boolean => Unit)(implicit executor: ExecutionContext): Unit = {
- jobHandle.addListener(new AbstractScalaJobHandleListener[T] {
- override def onJobCancelled(job: JobHandle[T]): Unit = {
- val onJobCancelledTask = new Runnable {
- override def run(): Unit = func(job.cancel(false))
- }
- executor.execute(onJobCancelledTask)
- }
- })
- }
-
- /**
- * Returns whether the job has already been completed with
- * a value or an exception.
- *
- * $nonDeterministic
- *
- * @return `true` if the job is already completed, `false` otherwise.
- */
- override def isCompleted: Boolean = jobHandle.isDone
-
- /**
- * The result value of the job.
- *
- * If the job is not completed the returned value will be `None`.
- * If the job is completed the value will be `Some(Success(t))`.
- * if it contains a valid result, or `Some(Failure(error))` if it contains
- * an exception.
- */
- override def value: Option[Try[T]] = {
- if (isCompleted) {
- Some(Try(getJavaFutureResult(jobHandle)))
- } else {
- None
- }
- }
-
- /**
- * Supports Scala's Await.result(atmost) which awaits the completion of the job and returns the
- * result (of type `T`).
- *
- * @param atMost
- * maximum wait time, which may be negative (no waiting is done),
- * [[scala.concurrent.duration.Duration.Inf Duration.Inf]] for unbounded waiting,
- * or a finite positive duration.
- * @return the result value if job is completed within the specific maximum wait time.
- * @throws Exception the underlying exception on the execution of the job.
- */
- @throws(classOf[Exception])
- override def result(atMost: Duration)(implicit permit: CanAwait): T =
- getJavaFutureResult(jobHandle, atMost)
-
- /**
- * Supports Scala's Await.ready(atmost) which awaits the completion of the job.
- *
- * @param atMost
- * maximum wait time, which may be negative (no waiting is done),
- * [[scala.concurrent.duration.Duration.Inf Duration.Inf]] for unbounded waiting,
- * or a finite positive duration.
- * @return ScalaJobHandle
- * @throws InterruptedException if the current thread is interrupted while waiting.
- * @throws TimeoutException if after waiting for the specified time the job
- * is still not ready.
- */
- @throws(classOf[InterruptedException])
- @throws(classOf[TimeoutException])
- override def ready(atMost: Duration)(implicit permit: CanAwait): ScalaJobHandle.this.type = {
- getJavaFutureResult(jobHandle, atMost)
- this
- }
-}
-
-private abstract class AbstractScalaJobHandleListener[T] extends Listener[T] {
- override def onJobQueued(job: JobHandle[T]): Unit = {}
-
- override def onJobCancelled(job: JobHandle[T]): Unit = {}
-
- override def onJobSucceeded(job: JobHandle[T], result: T): Unit = {}
-
- override def onJobStarted(job: JobHandle[T]): Unit = {}
-
- override def onJobFailed(job: JobHandle[T], cause: Throwable): Unit = {}
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/main/scala/com/cloudera/livy/scalaapi/package.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/main/scala/com/cloudera/livy/scalaapi/package.scala b/scala-api/src/main/scala/com/cloudera/livy/scalaapi/package.scala
deleted file mode 100644
index 71c8a7c..0000000
--- a/scala-api/src/main/scala/com/cloudera/livy/scalaapi/package.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy
-
-import java.util.concurrent.{ExecutionException, Future => JFuture, TimeUnit}
-
-import scala.concurrent.duration.Duration
-
-package object scalaapi {
-
- /**
- * A Scala Client for Livy which is a wrapper over the Java client.
- * @constructor Creates a Scala client.
- * @param livyJavaClient the Java client of Livy.
- * {{{
- * import com.cloudera.livy._
- * import com.cloudera.livy.scalaapi._
- * val url = "http://example.com"
- * val livyJavaClient = new LivyClientBuilder(false).setURI(new URI(url))).build()
- * val livyScalaClient = livyJavaClient.asScalaClient
- * }}}
- */
- implicit class ScalaWrapper(livyJavaClient: LivyClient) {
- def asScalaClient: LivyScalaClient = new LivyScalaClient(livyJavaClient)
- }
-
- private[livy] def getJavaFutureResult[T](jFuture: JFuture[T],
- atMost: Duration = Duration.Undefined): T = {
- try {
- if (!atMost.isFinite()) jFuture.get else jFuture.get(atMost.toMillis, TimeUnit.MILLISECONDS)
- } catch {
- case executionException: ExecutionException => throw executionException.getCause
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/main/scala/org/apache/livy/scalaapi/LivyScalaClient.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/main/scala/org/apache/livy/scalaapi/LivyScalaClient.scala b/scala-api/src/main/scala/org/apache/livy/scalaapi/LivyScalaClient.scala
new file mode 100644
index 0000000..ca7199a
--- /dev/null
+++ b/scala-api/src/main/scala/org/apache/livy/scalaapi/LivyScalaClient.scala
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.scalaapi
+
+import java.io.File
+import java.net.URI
+import java.util.concurrent.{Executors, Future => JFuture, ScheduledFuture, ThreadFactory, TimeUnit}
+
+import scala.concurrent._
+import scala.util.Try
+
+import org.apache.livy._
+
+/**
+ * A client for submitting Spark-based jobs to a Livy backend.
+ * @constructor Creates a Scala client.
+ * @param livyJavaClient the Java client of Livy.
+ */
+class LivyScalaClient(livyJavaClient: LivyClient) {
+
+ private val executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory {
+ override def newThread(r: Runnable): Thread = {
+ val thread = new Thread(r, "LivyScalaClient-PollingContainer")
+ thread.setDaemon(true)
+ thread
+ }
+ })
+
+ /**
+ * Submits a job for asynchronous execution.
+ *
+ * @param fn The job to be executed. It is a function that takes in a ScalaJobContext and
+ * returns the result of the execution of the job with that context.
+ * @return A handle that can be used to monitor the job.
+ */
+ def submit[T](fn: ScalaJobContext => T): ScalaJobHandle[T] = {
+ val job = new Job[T] {
+ @throws(classOf[Exception])
+ override def call(jobContext: JobContext): T = fn(new ScalaJobContext(jobContext))
+ }
+ new ScalaJobHandle(livyJavaClient.submit(job))
+ }
+
+ /**
+ * Asks the remote context to run a job immediately.
+ *
+ * Normally, the remote context will queue jobs and execute them based on how many worker
+ * threads have been configured. This method will run the submitted job in the same thread
+ * processing the RPC message, so that queueing does not apply.
+ *
+ * It's recommended that this method only be used to run code that finishes quickly. This
+ * avoids interfering with the normal operation of the context.
+ *
+ * @param fn The job to be executed. It is a function that takes in a ScalaJobContext and
+ * returns the result of the execution of the job with that context.
+ * @return A handle that can be used to monitor the job.
+ */
+ def run[T](fn: ScalaJobContext => T): Future[T] = {
+ val job = new Job[T] {
+ @throws(classOf[Exception])
+ override def call(jobContext: JobContext): T = {
+ val scalaJobContext = new ScalaJobContext(jobContext)
+ fn(scalaJobContext)
+ }
+ }
+ new PollingContainer(livyJavaClient.run(job)).poll()
+ }
+
+ /**
+ * Stops the remote context.
+ *
+ * Any pending jobs will be cancelled, and the remote context will be torn down.
+ *
+ * @param shutdownContext Whether to shutdown the underlying Spark context. If false, the
+ * context will keep running and it's still possible to send commands
+ * to it, if the backend being used supports it.
+ */
+ def stop(shutdownContext: Boolean): Unit = {
+ executor.shutdown()
+ livyJavaClient.stop(shutdownContext)
+ }
+
+ /**
+ * Upload a jar to be added to the Spark application classpath.
+ *
+ * @param jar The local file to be uploaded.
+ * @return A future that can be used to monitor this operation.
+ */
+ def uploadJar(jar: File): Future[_] = new PollingContainer(livyJavaClient.uploadJar(jar)).poll()
+
+ /**
+ * Adds a jar file to the running remote context.
+ *
+ * Note that the URL should be reachable by the Spark driver process. If running the driver
+ * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
+ * on that node (and not on the client machine).
+ *
+ * If the provided URI has no scheme, it's considered to be relative to the default file system
+ * configured in the Livy server.
+ *
+ * @param uri The location of the jar file.
+ * @return A future that can be used to monitor the operation.
+ */
+ def addJar(uri: URI): Future[_] = new PollingContainer(livyJavaClient.addJar(uri)).poll()
+
+ /**
+ * Upload a file to be passed to the Spark application.
+ *
+ * @param file The local file to be uploaded.
+ * @return A future that can be used to monitor this operation.
+ */
+ def uploadFile(file: File): Future[_] =
+ new PollingContainer(livyJavaClient.uploadFile(file)).poll()
+
+ /**
+ * Adds a file to the running remote context.
+ *
+ * Note that the URL should be reachable by the Spark driver process. If running the driver
+ * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
+ * on that node (and not on the client machine).
+ *
+ * If the provided URI has no scheme, it's considered to be relative to the default file system
+ * configured in the Livy server.
+ *
+ * @param uri The location of the file.
+ * @return A future that can be used to monitor the operation.
+ */
+ def addFile(uri: URI): Future[_] = new PollingContainer(livyJavaClient.addFile(uri)).poll()
+
+ private class PollingContainer[T] private[livy] (jFuture: JFuture[T]) extends Runnable {
+
+ private val initialDelay = 1
+ private val longDelay = 1
+ private var scheduledFuture: ScheduledFuture[_] = _
+ private val promise = Promise[T]
+
+ def poll(): Future[T] = {
+ scheduledFuture =
+ executor.scheduleWithFixedDelay(this, initialDelay, longDelay, TimeUnit.SECONDS)
+ promise.future
+ }
+
+ override def run(): Unit = {
+ if (jFuture.isDone) {
+ promise.complete(Try(getJavaFutureResult(jFuture)))
+ scheduledFuture.cancel(false)
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobContext.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobContext.scala b/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobContext.scala
new file mode 100644
index 0000000..ee082ec
--- /dev/null
+++ b/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobContext.scala
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.scalaapi
+
+import java.io.File
+
+import org.apache.spark.SparkContext
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.sql.hive.HiveContext
+import org.apache.spark.streaming.StreamingContext
+
+import org.apache.livy.JobContext
+
+/**
+ * Holds runtime information about the job execution context.
+ *
+ * @constructor Creates a ScalaJobContext.
+ * @param context the Java JobContext of Livy.
+ */
+class ScalaJobContext private[livy] (context: JobContext) {
+
+ /** The shared SparkContext instance. */
+ def sc: SparkContext = context.sc().sc
+
+ /** The shared SQLContext instance. */
+ def sqlctx: SQLContext = context.sqlctx()
+
+ /** The shared HiveContext instance. */
+ def hivectx: HiveContext = context.hivectx()
+
+ /** Returns the StreamingContext which has already been created. */
+ def streamingctx: StreamingContext = context.streamingctx().ssc
+
+ def sparkSession[E]: E = context.sparkSession()
+
+ /**
+ * Creates the SparkStreaming context.
+ *
+ * @param batchDuration Time interval at which streaming data will be divided into batches,
+ * in milliseconds.
+ */
+ def createStreamingContext(batchDuration: Long): Unit =
+ context.createStreamingContext(batchDuration)
+
+ /** Stops the SparkStreaming context. */
+ def stopStreamingContext(): Unit = context.stopStreamingCtx()
+
+ /**
+ * Returns a local tmp dir specific to the context.
+ */
+ def localTmpDir: File = context.getLocalTmpDir
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobHandle.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobHandle.scala b/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobHandle.scala
new file mode 100644
index 0000000..d1cf29d
--- /dev/null
+++ b/scala-api/src/main/scala/org/apache/livy/scalaapi/ScalaJobHandle.scala
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.scalaapi
+
+import scala.concurrent.{CanAwait, ExecutionContext, Future, TimeoutException}
+import scala.concurrent.duration.Duration
+import scala.util.Try
+
+import org.apache.livy.JobHandle
+import org.apache.livy.JobHandle.{Listener, State}
+
+/**
+ * A handle to a submitted job. Allows for monitoring and controlling of the running remote job.
+ *
+ * @constructor Creates a ScalaJobHandle.
+ * @param jobHandle the Java JobHandle of Livy.
+ *
+ * @define multipleCallbacks
+ * Multiple callbacks may be registered; there is no guarantee that they will be
+ * executed in a particular order.
+ *
+ * @define nonDeterministic
+ * Note: using this method yields nondeterministic dataflow programs.
+ *
+ * @define callbackInContext
+ * The provided callback always runs in the provided implicit
+ *` ExecutionContext`, though there is no guarantee that the
+ * `execute()` method on the `ExecutionContext` will be called once
+ * per callback or that `execute()` will be called in the current
+ * thread. That is, the implementation may run multiple callbacks
+ * in a batch within a single `execute()` and it may run
+ * `execute()` either immediately or asynchronously.
+ */
+class ScalaJobHandle[T] private[livy] (jobHandle: JobHandle[T]) extends Future[T] {
+
+ /**
+ * Return the current state of the job.
+ */
+ def state: State = jobHandle.getState()
+
+ /**
+ * When the job is completed, either through an exception, or a value,
+ * apply the provided function.
+ *
+ * If the job has already been completed,
+ * this will either be applied immediately or be scheduled asynchronously.
+ *
+ * $multipleCallbacks
+ * $callbackInContext
+ */
+ override def onComplete[U](func: (Try[T]) => U)(implicit executor: ExecutionContext): Unit = {
+ jobHandle.addListener(new AbstractScalaJobHandleListener[T] {
+ override def onJobSucceeded(job: JobHandle[T], result: T): Unit = {
+ val onJobSucceededTask = new Runnable {
+ override def run(): Unit = func(Try(result))
+ }
+ executor.execute(onJobSucceededTask)
+ }
+
+ override def onJobFailed(job: JobHandle[T], cause: Throwable): Unit = {
+ val onJobFailedTask = new Runnable {
+ override def run(): Unit = func(Try(getJavaFutureResult(job)))
+ }
+ executor.execute(onJobFailedTask)
+ }
+ })
+ }
+
+ /**
+ * When this job is queued, apply the provided function.
+ *
+ * $multipleCallbacks
+ * $callbackInContext
+ */
+ def onJobQueued[U](func: => Unit)(implicit executor: ExecutionContext): Unit = {
+ jobHandle.addListener(new AbstractScalaJobHandleListener[T] {
+ override def onJobQueued(job: JobHandle[T]): Unit = {
+ val onJobQueuedTask = new Runnable {
+ override def run(): Unit = func
+ }
+ executor.execute(onJobQueuedTask)
+ }
+ })
+ }
+
+ /**
+ * When this job has started, apply the provided function.
+ *
+ * $multipleCallbacks
+ * $callbackInContext
+ */
+ def onJobStarted[U](func: => Unit)(implicit executor: ExecutionContext): Unit = {
+ jobHandle.addListener(new AbstractScalaJobHandleListener[T] {
+ override def onJobStarted(job: JobHandle[T]): Unit = {
+ val onJobStartedTask = new Runnable {
+ override def run(): Unit = func
+ }
+ executor.execute(onJobStartedTask)
+ }
+ })
+ }
+
+ /**
+ * When this job is cancelled, apply the provided function.
+ *
+ * $multipleCallbacks
+ * $callbackInContext
+ */
+ def onJobCancelled[U](func: Boolean => Unit)(implicit executor: ExecutionContext): Unit = {
+ jobHandle.addListener(new AbstractScalaJobHandleListener[T] {
+ override def onJobCancelled(job: JobHandle[T]): Unit = {
+ val onJobCancelledTask = new Runnable {
+ override def run(): Unit = func(job.cancel(false))
+ }
+ executor.execute(onJobCancelledTask)
+ }
+ })
+ }
+
+ /**
+ * Returns whether the job has already been completed with
+ * a value or an exception.
+ *
+ * $nonDeterministic
+ *
+ * @return `true` if the job is already completed, `false` otherwise.
+ */
+ override def isCompleted: Boolean = jobHandle.isDone
+
+ /**
+ * The result value of the job.
+ *
+ * If the job is not completed the returned value will be `None`.
+ * If the job is completed the value will be `Some(Success(t))`.
+ * if it contains a valid result, or `Some(Failure(error))` if it contains
+ * an exception.
+ */
+ override def value: Option[Try[T]] = {
+ if (isCompleted) {
+ Some(Try(getJavaFutureResult(jobHandle)))
+ } else {
+ None
+ }
+ }
+
+ /**
+ * Supports Scala's Await.result(atmost) which awaits the completion of the job and returns the
+ * result (of type `T`).
+ *
+ * @param atMost
+ * maximum wait time, which may be negative (no waiting is done),
+ * [[scala.concurrent.duration.Duration.Inf Duration.Inf]] for unbounded waiting,
+ * or a finite positive duration.
+ * @return the result value if job is completed within the specific maximum wait time.
+ * @throws Exception the underlying exception on the execution of the job.
+ */
+ @throws(classOf[Exception])
+ override def result(atMost: Duration)(implicit permit: CanAwait): T =
+ getJavaFutureResult(jobHandle, atMost)
+
+ /**
+ * Supports Scala's Await.ready(atmost) which awaits the completion of the job.
+ *
+ * @param atMost
+ * maximum wait time, which may be negative (no waiting is done),
+ * [[scala.concurrent.duration.Duration.Inf Duration.Inf]] for unbounded waiting,
+ * or a finite positive duration.
+ * @return ScalaJobHandle
+ * @throws InterruptedException if the current thread is interrupted while waiting.
+ * @throws TimeoutException if after waiting for the specified time the job
+ * is still not ready.
+ */
+ @throws(classOf[InterruptedException])
+ @throws(classOf[TimeoutException])
+ override def ready(atMost: Duration)(implicit permit: CanAwait): ScalaJobHandle.this.type = {
+ getJavaFutureResult(jobHandle, atMost)
+ this
+ }
+}
+
+private abstract class AbstractScalaJobHandleListener[T] extends Listener[T] {
+ override def onJobQueued(job: JobHandle[T]): Unit = {}
+
+ override def onJobCancelled(job: JobHandle[T]): Unit = {}
+
+ override def onJobSucceeded(job: JobHandle[T], result: T): Unit = {}
+
+ override def onJobStarted(job: JobHandle[T]): Unit = {}
+
+ override def onJobFailed(job: JobHandle[T], cause: Throwable): Unit = {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/main/scala/org/apache/livy/scalaapi/package.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/main/scala/org/apache/livy/scalaapi/package.scala b/scala-api/src/main/scala/org/apache/livy/scalaapi/package.scala
new file mode 100644
index 0000000..6e53a37
--- /dev/null
+++ b/scala-api/src/main/scala/org/apache/livy/scalaapi/package.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy
+
+import java.util.concurrent.{ExecutionException, Future => JFuture, TimeUnit}
+
+import scala.concurrent.duration.Duration
+
+package object scalaapi {
+
+ /**
+ * A Scala Client for Livy which is a wrapper over the Java client.
+ * @constructor Creates a Scala client.
+ * @param livyJavaClient the Java client of Livy.
+ * {{{
+ * import com.cloudera.livy._
+ * import com.cloudera.livy.scalaapi._
+ * val url = "http://example.com"
+ * val livyJavaClient = new LivyClientBuilder(false).setURI(new URI(url))).build()
+ * val livyScalaClient = livyJavaClient.asScalaClient
+ * }}}
+ */
+ implicit class ScalaWrapper(livyJavaClient: LivyClient) {
+ def asScalaClient: LivyScalaClient = new LivyScalaClient(livyJavaClient)
+ }
+
+ private[livy] def getJavaFutureResult[T](jFuture: JFuture[T],
+ atMost: Duration = Duration.Undefined): T = {
+ try {
+ if (!atMost.isFinite()) jFuture.get else jFuture.get(atMost.toMillis, TimeUnit.MILLISECONDS)
+ } catch {
+ case executionException: ExecutionException => throw executionException.getCause
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaClientTest.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaClientTest.scala b/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaClientTest.scala
deleted file mode 100644
index 1bf879c..0000000
--- a/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaClientTest.scala
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.scalaapi
-
-import java.io._
-import java.net.URI
-import java.nio.charset.StandardCharsets._
-import java.util._
-import java.util.concurrent.CountDownLatch
-import java.util.jar.JarOutputStream
-import java.util.zip.ZipEntry
-
-import scala.concurrent.Await
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.duration._
-import scala.language.postfixOps
-import scala.util.{Failure, Success}
-
-import org.apache.spark.SparkFiles
-import org.apache.spark.launcher.SparkLauncher
-import org.scalatest.{BeforeAndAfter, FunSuite}
-import org.scalatest.concurrent.ScalaFutures
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-import com.cloudera.livy.rsc.RSCConf.Entry._
-
-class ScalaClientTest extends FunSuite
- with ScalaFutures
- with BeforeAndAfter
- with LivyBaseUnitTestSuite {
-
- import com.cloudera.livy._
-
- private var client: LivyScalaClient = _
-
- after {
- if (client != null) {
- client.stop(true)
- client = null
- }
- }
-
- test("test Job Submission") {
- configureClient(true)
- val jobHandle = client.submit(ScalaClientTestUtils.helloJob)
- ScalaClientTestUtils.assertTestPassed(jobHandle, "hello")
- }
-
- test("test Simple Spark Job") {
- configureClient(true)
- val sFuture = client.submit(ScalaClientTestUtils.simpleSparkJob)
- ScalaClientTestUtils.assertTestPassed(sFuture, 5)
- }
-
- test("test Job Failure") {
- configureClient(true)
- val sFuture = client.submit(ScalaClientTestUtils.throwExceptionJob)
- val lock = new CountDownLatch(1)
- var testFailure : Option[String] = None
- sFuture onComplete {
- case Success(t) => {
- testFailure = Some("Test should have thrown CustomFailureException")
- lock.countDown()
- }
- case Failure(e) => {
- if (!e.getMessage.contains("CustomTestFailureException")) {
- testFailure = Some("Test did not throw expected exception - CustomFailureException")
- }
- lock.countDown()
- }
- }
- ScalaClientTestUtils.assertAwait(lock)
- testFailure.foreach(fail(_))
- }
-
- test("test Sync Rpc") {
- configureClient(true)
- val future = client.run(ScalaClientTestUtils.helloJob)
- ScalaClientTestUtils.assertTestPassed(future, "hello")
- }
-
- test("test Remote client") {
- configureClient(false)
- val sFuture = client.submit(ScalaClientTestUtils.simpleSparkJob)
- ScalaClientTestUtils.assertTestPassed(sFuture, 5)
- }
-
- test("test add file") {
- configureClient(true)
- val file = File.createTempFile("test", ".file")
- val fileStream = new FileOutputStream(file)
- fileStream.write("test file".getBytes("UTF-8"))
- fileStream.close
- val addFileFuture = client.addFile(new URI("file:" + file.getAbsolutePath()))
- Await.ready(addFileFuture, ScalaClientTestUtils.Timeout second)
- val sFuture = client.submit { context =>
- ScalaClientTest.fileOperation(false, file.getName, context)
- }
- ScalaClientTestUtils.assertTestPassed(sFuture, "test file")
- }
-
- test("test add jar") {
- configureClient(true)
- val jar = File.createTempFile("test", ".resource")
- val jarFile = new JarOutputStream(new FileOutputStream(jar))
- jarFile.putNextEntry(new ZipEntry("test.resource"))
- jarFile.write("test resource".getBytes("UTF-8"))
- jarFile.closeEntry()
- jarFile.close()
- val addJarFuture = client.addJar(new URI("file:" + jar.getAbsolutePath()))
- Await.ready(addJarFuture, ScalaClientTestUtils.Timeout second)
- val sFuture = client.submit { context =>
- ScalaClientTest.fileOperation(true, "test.resource", context)
- }
- ScalaClientTestUtils.assertTestPassed(sFuture, "test resource")
- }
-
- test("Successive onComplete callbacks") {
- var testFailure: Option[String] = None
- configureClient(true)
- val future = client.run(ScalaClientTestUtils.helloJob)
- val lock = new CountDownLatch(3)
- for (i <- 0 to 2) {
- future onComplete {
- case Success(t) => {
- if (!t.equals("hello")) testFailure = Some("Expected message not returned")
- lock.countDown()
- }
- case Failure(e) => {
- testFailure = Some("onComplete should not have triggered Failure callback")
- lock.countDown()
- }
- }
- }
- ScalaClientTestUtils.assertAwait(lock)
- testFailure.foreach(fail(_))
- }
-
- private def configureClient(local: Boolean) = {
- val conf = ScalaClientTest.createConf(local)
- val javaClient = new LivyClientBuilder(false).setURI(new URI("rsc:/")).setAll(conf).build()
- client = javaClient.asScalaClient
- pingJob()
- }
-
- private def pingJob() = {
- val future = client.submit { context =>
- null
- }
- ScalaClientTestUtils.assertTestPassed(future, null)
- }
-}
-
-class CustomTestFailureException extends RuntimeException {}
-
-object ScalaClientTest {
-
- def createConf(local: Boolean): Properties = {
- val conf = new Properties
- if (local) {
- conf.put(CLIENT_IN_PROCESS.key, "true")
- conf.put(SparkLauncher.SPARK_MASTER, "local")
- conf.put("spark.app.name", "SparkClientSuite Local App")
- } else {
- val classpath: String = System.getProperty("java.class.path")
- conf.put("spark.app.name", "SparkClientSuite Remote App")
- conf.put(SparkLauncher.DRIVER_MEMORY, "512m")
- conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, classpath)
- conf.put(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH, classpath)
- }
- conf.put(LIVY_JARS.key, "")
- conf
- }
-
- def fileOperation(isResource: Boolean, fileName: String, context: ScalaJobContext): String = {
- val arr = Seq(1)
- val rdd = context.sc.parallelize(arr).map { value =>
- var inputStream: InputStream = null
- if (isResource) {
- val ccl = Thread.currentThread.getContextClassLoader
- inputStream = ccl.getResourceAsStream(fileName)
- } else {
- inputStream = new FileInputStream(SparkFiles.get(fileName))
- }
- try {
- val out = new ByteArrayOutputStream()
- val buffer = new Array[Byte](1024)
- var read = inputStream.read(buffer)
- while (read >= 0) {
- out.write(buffer, 0, read)
- read = inputStream.read(buffer)
- }
- val bytes = out.toByteArray
- new String(bytes, 0, bytes.length, UTF_8)
- } finally {
- inputStream.close()
- }
- }
- rdd.collect().head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaClientTestUtils.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaClientTestUtils.scala b/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaClientTestUtils.scala
deleted file mode 100644
index 8727db5..0000000
--- a/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaClientTestUtils.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.scalaapi
-
-import java.util.Random
-import java.util.concurrent.{CountDownLatch, TimeUnit}
-
-import scala.collection.mutable.ArrayBuffer
-import scala.concurrent.{Await, Future}
-import scala.concurrent.duration._
-
-import org.scalatest.FunSuite
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-
-object ScalaClientTestUtils extends FunSuite with LivyBaseUnitTestSuite {
-
- val Timeout = 40
-
- def helloJob(context: ScalaJobContext): String = "hello"
-
- def throwExceptionJob(context: ScalaJobContext): Unit = throw new CustomTestFailureException
-
- def simpleSparkJob(context: ScalaJobContext): Long = {
- val r = new Random
- val count = 5
- val partitions = Math.min(r.nextInt(10) + 1, count)
- val buffer = new ArrayBuffer[Int]()
- for (a <- 1 to count) {
- buffer += r.nextInt()
- }
- context.sc.parallelize(buffer, partitions).count()
- }
-
- def assertAwait(lock: CountDownLatch): Unit = {
- assert(lock.await(Timeout, TimeUnit.SECONDS) == true)
- }
-
- def assertTestPassed[T](future: Future[T], expectedValue: T): Unit = {
- val result = Await.result(future, Timeout second)
- assert(result === expectedValue)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaJobHandleTest.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaJobHandleTest.scala b/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaJobHandleTest.scala
deleted file mode 100644
index 7dd9444..0000000
--- a/scala-api/src/test/scala/com/cloudera/livy/scalaapi/ScalaJobHandleTest.scala
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.scalaapi
-
-import java.util.concurrent.{CountDownLatch, TimeUnit}
-
-import scala.concurrent.Await
-import scala.concurrent.duration._
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.language.postfixOps
-import scala.util.{Failure, Success}
-
-import org.mockito.Matchers._
-import org.mockito.Mockito._
-import org.scalatest.{BeforeAndAfter, FunSuite}
-import org.scalatest.concurrent.ScalaFutures
-
-import com.cloudera.livy.{JobHandle, LivyBaseUnitTestSuite}
-import com.cloudera.livy.JobHandle.{Listener, State}
-
-class ScalaJobHandleTest extends FunSuite
- with ScalaFutures
- with BeforeAndAfter
- with LivyBaseUnitTestSuite {
-
- private var mockJobHandle: JobHandle[String] = null
- private var scalaJobHandle: ScalaJobHandle[String] = null
- private val timeoutInMilliseconds = 5000
- private var listener: JobHandle.Listener[String] = null
-
- before {
- listener = mock(classOf[JobHandle.Listener[String]])
- mockJobHandle = mock(classOf[JobHandle[String]])
- scalaJobHandle = new ScalaJobHandle(mockJobHandle)
- }
-
- test("get result when job is already complete") {
- when(mockJobHandle.get(timeoutInMilliseconds, TimeUnit.MILLISECONDS)).thenReturn("hello")
- val result = Await.result(scalaJobHandle, 5 seconds)
- assert(result == "hello")
- verify(mockJobHandle, times(1)).get(timeoutInMilliseconds, TimeUnit.MILLISECONDS)
- }
-
- test("ready when the thread waits for the mentioned duration for job to complete") {
- when(mockJobHandle.get(timeoutInMilliseconds, TimeUnit.MILLISECONDS)).thenReturn("hello")
- val result = Await.ready(scalaJobHandle, 5 seconds)
- assert(result == scalaJobHandle)
- verify(mockJobHandle, times(1)).get(timeoutInMilliseconds, TimeUnit.MILLISECONDS)
- }
-
- test("ready with Infinite Duration") {
- when(mockJobHandle.isDone).thenReturn(true)
- when(mockJobHandle.get()).thenReturn("hello")
- val result = Await.ready(scalaJobHandle, Duration.Undefined)
- assert(result == scalaJobHandle)
- verify(mockJobHandle, times(1)).get()
- }
-
- test("verify addListener call of java jobHandle for onComplete") {
- doNothing().when(mockJobHandle).addListener(listener)
- scalaJobHandle onComplete {
- case Success(t) => {}
- }
- verify(mockJobHandle).addListener(isA(classOf[Listener[String]]))
- verify(mockJobHandle, times(1)).addListener(any())
- }
-
- test("onComplete Success") {
- val jobHandleStub = new AbstractJobHandleStub[String] {
- override def addListener(l: Listener[String]): Unit = l.onJobSucceeded(this, "hello")
- }
- val lock = new CountDownLatch(1)
- var testFailure: Option[String] = None
- val testScalaHandle = new ScalaJobHandle(jobHandleStub)
- testScalaHandle onComplete {
- case Success(t) => {
- if (!t.equals("hello")) {
- testFailure = Some("onComplete has not returned the expected message")
- }
- lock.countDown()
- }
- case Failure(e) => {
- testFailure = Some("onComplete should not have triggered Failure callback")
- lock.countDown()
- }
- }
- ScalaClientTestUtils.assertAwait(lock)
- testFailure.foreach(fail(_))
- }
-
- test("onComplete Failure") {
- val jobHandleStub = new AbstractJobHandleStub[String] {
- override def addListener(l: Listener[String]): Unit =
- l.onJobFailed(this, new CustomTestFailureException)
-
- override def get(): String = throw new CustomTestFailureException()
- }
- val lock = new CountDownLatch(1)
- var testFailure: Option[String] = None
- val testScalaHandle = new ScalaJobHandle(jobHandleStub)
- testScalaHandle onComplete {
- case Success(t) => {
- testFailure = Some("Test should have thrown CustomFailureException")
- lock.countDown()
- }
- case Failure(e) => {
- if (!e.isInstanceOf[CustomTestFailureException]) {
- testFailure = Some("Test did not throw expected exception - CustomFailureException")
- }
- lock.countDown()
- }
- }
- ScalaClientTestUtils.assertAwait(lock)
- testFailure.foreach(fail(_))
- }
-
- test("onJobCancelled") {
- val jobHandleStub = new AbstractJobHandleStub[String] {
- override def addListener(l: Listener[String]): Unit = l.onJobCancelled(this)
- override def cancel(mayInterruptIfRunning: Boolean): Boolean = true
- }
- var testFailure: Option[String] = None
- val lock = new CountDownLatch(1)
- val testScalaHandle = new ScalaJobHandle(jobHandleStub)
- testScalaHandle onJobCancelled {
- case true => lock.countDown()
- case false => {
- testFailure = Some("False callback should not have been triggered")
- lock.countDown()
- }
- }
- ScalaClientTestUtils.assertAwait(lock)
- testFailure.foreach(fail(_))
- }
-
- test("onJobQueued") {
- val jobHandleStub = new AbstractJobHandleStub[String] {
- override def addListener(l: Listener[String]): Unit = l.onJobQueued(this)
- }
- val lock = new CountDownLatch(1)
- val testScalaHandle = new ScalaJobHandle(jobHandleStub)
- testScalaHandle onJobQueued {
- lock.countDown()
- }
- ScalaClientTestUtils.assertAwait(lock)
- }
-
- test("onJobStarted") {
- val jobHandleStub = new AbstractJobHandleStub[String] {
- override def addListener(l: Listener[String]): Unit = l.onJobStarted(this)
- }
- val lock = new CountDownLatch(1)
- val testScalaHandle = new ScalaJobHandle(jobHandleStub)
- testScalaHandle onJobStarted {
- lock.countDown()
- }
- ScalaClientTestUtils.assertAwait(lock)
- }
-}
-
-private abstract class AbstractJobHandleStub[T] private[livy] extends JobHandle[T] {
-
- override def getState: State = null
-
- override def addListener(l: Listener[T]): Unit = {}
-
- override def isCancelled: Boolean = false
-
- override def get(): T = null.asInstanceOf[T]
-
- override def get(timeout: Long, unit: TimeUnit): T = null.asInstanceOf[T]
-
- override def cancel(mayInterruptIfRunning: Boolean): Boolean = false
-
- override def isDone: Boolean = true
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTest.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTest.scala b/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTest.scala
new file mode 100644
index 0000000..79760b7
--- /dev/null
+++ b/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTest.scala
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.scalaapi
+
+import java.io._
+import java.net.URI
+import java.nio.charset.StandardCharsets._
+import java.util._
+import java.util.concurrent.CountDownLatch
+import java.util.jar.JarOutputStream
+import java.util.zip.ZipEntry
+
+import scala.concurrent.Await
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.util.{Failure, Success}
+
+import org.apache.spark.SparkFiles
+import org.apache.spark.launcher.SparkLauncher
+import org.scalatest.{BeforeAndAfter, FunSuite}
+import org.scalatest.concurrent.ScalaFutures
+
+import org.apache.livy.LivyBaseUnitTestSuite
+import org.apache.livy.rsc.RSCConf.Entry._
+
+class ScalaClientTest extends FunSuite
+ with ScalaFutures
+ with BeforeAndAfter
+ with LivyBaseUnitTestSuite {
+
+ import org.apache.livy._
+
+ private var client: LivyScalaClient = _
+
+ after {
+ if (client != null) {
+ client.stop(true)
+ client = null
+ }
+ }
+
+ test("test Job Submission") {
+ configureClient(true)
+ val jobHandle = client.submit(ScalaClientTestUtils.helloJob)
+ ScalaClientTestUtils.assertTestPassed(jobHandle, "hello")
+ }
+
+ test("test Simple Spark Job") {
+ configureClient(true)
+ val sFuture = client.submit(ScalaClientTestUtils.simpleSparkJob)
+ ScalaClientTestUtils.assertTestPassed(sFuture, 5)
+ }
+
+ test("test Job Failure") {
+ configureClient(true)
+ val sFuture = client.submit(ScalaClientTestUtils.throwExceptionJob)
+ val lock = new CountDownLatch(1)
+ var testFailure : Option[String] = None
+ sFuture onComplete {
+ case Success(t) => {
+ testFailure = Some("Test should have thrown CustomFailureException")
+ lock.countDown()
+ }
+ case Failure(e) => {
+ if (!e.getMessage.contains("CustomTestFailureException")) {
+ testFailure = Some("Test did not throw expected exception - CustomFailureException")
+ }
+ lock.countDown()
+ }
+ }
+ ScalaClientTestUtils.assertAwait(lock)
+ testFailure.foreach(fail(_))
+ }
+
+ test("test Sync Rpc") {
+ configureClient(true)
+ val future = client.run(ScalaClientTestUtils.helloJob)
+ ScalaClientTestUtils.assertTestPassed(future, "hello")
+ }
+
+ test("test Remote client") {
+ configureClient(false)
+ val sFuture = client.submit(ScalaClientTestUtils.simpleSparkJob)
+ ScalaClientTestUtils.assertTestPassed(sFuture, 5)
+ }
+
+ test("test add file") {
+ configureClient(true)
+ val file = File.createTempFile("test", ".file")
+ val fileStream = new FileOutputStream(file)
+ fileStream.write("test file".getBytes("UTF-8"))
+ fileStream.close
+ val addFileFuture = client.addFile(new URI("file:" + file.getAbsolutePath()))
+ Await.ready(addFileFuture, ScalaClientTestUtils.Timeout second)
+ val sFuture = client.submit { context =>
+ ScalaClientTest.fileOperation(false, file.getName, context)
+ }
+ ScalaClientTestUtils.assertTestPassed(sFuture, "test file")
+ }
+
+ test("test add jar") {
+ configureClient(true)
+ val jar = File.createTempFile("test", ".resource")
+ val jarFile = new JarOutputStream(new FileOutputStream(jar))
+ jarFile.putNextEntry(new ZipEntry("test.resource"))
+ jarFile.write("test resource".getBytes("UTF-8"))
+ jarFile.closeEntry()
+ jarFile.close()
+ val addJarFuture = client.addJar(new URI("file:" + jar.getAbsolutePath()))
+ Await.ready(addJarFuture, ScalaClientTestUtils.Timeout second)
+ val sFuture = client.submit { context =>
+ ScalaClientTest.fileOperation(true, "test.resource", context)
+ }
+ ScalaClientTestUtils.assertTestPassed(sFuture, "test resource")
+ }
+
+ test("Successive onComplete callbacks") {
+ var testFailure: Option[String] = None
+ configureClient(true)
+ val future = client.run(ScalaClientTestUtils.helloJob)
+ val lock = new CountDownLatch(3)
+ for (i <- 0 to 2) {
+ future onComplete {
+ case Success(t) => {
+ if (!t.equals("hello")) testFailure = Some("Expected message not returned")
+ lock.countDown()
+ }
+ case Failure(e) => {
+ testFailure = Some("onComplete should not have triggered Failure callback")
+ lock.countDown()
+ }
+ }
+ }
+ ScalaClientTestUtils.assertAwait(lock)
+ testFailure.foreach(fail(_))
+ }
+
+ private def configureClient(local: Boolean) = {
+ val conf = ScalaClientTest.createConf(local)
+ val javaClient = new LivyClientBuilder(false).setURI(new URI("rsc:/")).setAll(conf).build()
+ client = javaClient.asScalaClient
+ pingJob()
+ }
+
+ private def pingJob() = {
+ val future = client.submit { context =>
+ null
+ }
+ ScalaClientTestUtils.assertTestPassed(future, null)
+ }
+}
+
+class CustomTestFailureException extends RuntimeException {}
+
+object ScalaClientTest {
+
+ def createConf(local: Boolean): Properties = {
+ val conf = new Properties
+ if (local) {
+ conf.put(CLIENT_IN_PROCESS.key, "true")
+ conf.put(SparkLauncher.SPARK_MASTER, "local")
+ conf.put("spark.app.name", "SparkClientSuite Local App")
+ } else {
+ val classpath: String = System.getProperty("java.class.path")
+ conf.put("spark.app.name", "SparkClientSuite Remote App")
+ conf.put(SparkLauncher.DRIVER_MEMORY, "512m")
+ conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, classpath)
+ conf.put(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH, classpath)
+ }
+ conf.put(LIVY_JARS.key, "")
+ conf
+ }
+
+ def fileOperation(isResource: Boolean, fileName: String, context: ScalaJobContext): String = {
+ val arr = Seq(1)
+ val rdd = context.sc.parallelize(arr).map { value =>
+ var inputStream: InputStream = null
+ if (isResource) {
+ val ccl = Thread.currentThread.getContextClassLoader
+ inputStream = ccl.getResourceAsStream(fileName)
+ } else {
+ inputStream = new FileInputStream(SparkFiles.get(fileName))
+ }
+ try {
+ val out = new ByteArrayOutputStream()
+ val buffer = new Array[Byte](1024)
+ var read = inputStream.read(buffer)
+ while (read >= 0) {
+ out.write(buffer, 0, read)
+ read = inputStream.read(buffer)
+ }
+ val bytes = out.toByteArray
+ new String(bytes, 0, bytes.length, UTF_8)
+ } finally {
+ inputStream.close()
+ }
+ }
+ rdd.collect().head
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTestUtils.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTestUtils.scala b/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTestUtils.scala
new file mode 100644
index 0000000..458ff3b
--- /dev/null
+++ b/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTestUtils.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.scalaapi
+
+import java.util.Random
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+import scala.collection.mutable.ArrayBuffer
+import scala.concurrent.{Await, Future}
+import scala.concurrent.duration._
+
+import org.scalatest.FunSuite
+
+import org.apache.livy.LivyBaseUnitTestSuite
+
+object ScalaClientTestUtils extends FunSuite with LivyBaseUnitTestSuite {
+
+ val Timeout = 40
+
+ def helloJob(context: ScalaJobContext): String = "hello"
+
+ def throwExceptionJob(context: ScalaJobContext): Unit = throw new CustomTestFailureException
+
+ def simpleSparkJob(context: ScalaJobContext): Long = {
+ val r = new Random
+ val count = 5
+ val partitions = Math.min(r.nextInt(10) + 1, count)
+ val buffer = new ArrayBuffer[Int]()
+ for (a <- 1 to count) {
+ buffer += r.nextInt()
+ }
+ context.sc.parallelize(buffer, partitions).count()
+ }
+
+ def assertAwait(lock: CountDownLatch): Unit = {
+ assert(lock.await(Timeout, TimeUnit.SECONDS) == true)
+ }
+
+ def assertTestPassed[T](future: Future[T], expectedValue: T): Unit = {
+ val result = Await.result(future, Timeout second)
+ assert(result === expectedValue)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaJobHandleTest.scala
----------------------------------------------------------------------
diff --git a/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaJobHandleTest.scala b/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaJobHandleTest.scala
new file mode 100644
index 0000000..3f5bdc6
--- /dev/null
+++ b/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaJobHandleTest.scala
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.scalaapi
+
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+import scala.concurrent.Await
+import scala.concurrent.duration._
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.language.postfixOps
+import scala.util.{Failure, Success}
+
+import org.mockito.Matchers._
+import org.mockito.Mockito._
+import org.scalatest.{BeforeAndAfter, FunSuite}
+import org.scalatest.concurrent.ScalaFutures
+
+import org.apache.livy.{JobHandle, LivyBaseUnitTestSuite}
+import org.apache.livy.JobHandle.{Listener, State}
+
+class ScalaJobHandleTest extends FunSuite
+ with ScalaFutures
+ with BeforeAndAfter
+ with LivyBaseUnitTestSuite {
+
+ private var mockJobHandle: JobHandle[String] = null
+ private var scalaJobHandle: ScalaJobHandle[String] = null
+ private val timeoutInMilliseconds = 5000
+ private var listener: JobHandle.Listener[String] = null
+
+ before {
+ listener = mock(classOf[JobHandle.Listener[String]])
+ mockJobHandle = mock(classOf[JobHandle[String]])
+ scalaJobHandle = new ScalaJobHandle(mockJobHandle)
+ }
+
+ test("get result when job is already complete") {
+ when(mockJobHandle.get(timeoutInMilliseconds, TimeUnit.MILLISECONDS)).thenReturn("hello")
+ val result = Await.result(scalaJobHandle, 5 seconds)
+ assert(result == "hello")
+ verify(mockJobHandle, times(1)).get(timeoutInMilliseconds, TimeUnit.MILLISECONDS)
+ }
+
+ test("ready when the thread waits for the mentioned duration for job to complete") {
+ when(mockJobHandle.get(timeoutInMilliseconds, TimeUnit.MILLISECONDS)).thenReturn("hello")
+ val result = Await.ready(scalaJobHandle, 5 seconds)
+ assert(result == scalaJobHandle)
+ verify(mockJobHandle, times(1)).get(timeoutInMilliseconds, TimeUnit.MILLISECONDS)
+ }
+
+ test("ready with Infinite Duration") {
+ when(mockJobHandle.isDone).thenReturn(true)
+ when(mockJobHandle.get()).thenReturn("hello")
+ val result = Await.ready(scalaJobHandle, Duration.Undefined)
+ assert(result == scalaJobHandle)
+ verify(mockJobHandle, times(1)).get()
+ }
+
+ test("verify addListener call of java jobHandle for onComplete") {
+ doNothing().when(mockJobHandle).addListener(listener)
+ scalaJobHandle onComplete {
+ case Success(t) => {}
+ }
+ verify(mockJobHandle).addListener(isA(classOf[Listener[String]]))
+ verify(mockJobHandle, times(1)).addListener(any())
+ }
+
+ test("onComplete Success") {
+ val jobHandleStub = new AbstractJobHandleStub[String] {
+ override def addListener(l: Listener[String]): Unit = l.onJobSucceeded(this, "hello")
+ }
+ val lock = new CountDownLatch(1)
+ var testFailure: Option[String] = None
+ val testScalaHandle = new ScalaJobHandle(jobHandleStub)
+ testScalaHandle onComplete {
+ case Success(t) => {
+ if (!t.equals("hello")) {
+ testFailure = Some("onComplete has not returned the expected message")
+ }
+ lock.countDown()
+ }
+ case Failure(e) => {
+ testFailure = Some("onComplete should not have triggered Failure callback")
+ lock.countDown()
+ }
+ }
+ ScalaClientTestUtils.assertAwait(lock)
+ testFailure.foreach(fail(_))
+ }
+
+ test("onComplete Failure") {
+ val jobHandleStub = new AbstractJobHandleStub[String] {
+ override def addListener(l: Listener[String]): Unit =
+ l.onJobFailed(this, new CustomTestFailureException)
+
+ override def get(): String = throw new CustomTestFailureException()
+ }
+ val lock = new CountDownLatch(1)
+ var testFailure: Option[String] = None
+ val testScalaHandle = new ScalaJobHandle(jobHandleStub)
+ testScalaHandle onComplete {
+ case Success(t) => {
+ testFailure = Some("Test should have thrown CustomFailureException")
+ lock.countDown()
+ }
+ case Failure(e) => {
+ if (!e.isInstanceOf[CustomTestFailureException]) {
+ testFailure = Some("Test did not throw expected exception - CustomFailureException")
+ }
+ lock.countDown()
+ }
+ }
+ ScalaClientTestUtils.assertAwait(lock)
+ testFailure.foreach(fail(_))
+ }
+
+ test("onJobCancelled") {
+ val jobHandleStub = new AbstractJobHandleStub[String] {
+ override def addListener(l: Listener[String]): Unit = l.onJobCancelled(this)
+ override def cancel(mayInterruptIfRunning: Boolean): Boolean = true
+ }
+ var testFailure: Option[String] = None
+ val lock = new CountDownLatch(1)
+ val testScalaHandle = new ScalaJobHandle(jobHandleStub)
+ testScalaHandle onJobCancelled {
+ case true => lock.countDown()
+ case false => {
+ testFailure = Some("False callback should not have been triggered")
+ lock.countDown()
+ }
+ }
+ ScalaClientTestUtils.assertAwait(lock)
+ testFailure.foreach(fail(_))
+ }
+
+ test("onJobQueued") {
+ val jobHandleStub = new AbstractJobHandleStub[String] {
+ override def addListener(l: Listener[String]): Unit = l.onJobQueued(this)
+ }
+ val lock = new CountDownLatch(1)
+ val testScalaHandle = new ScalaJobHandle(jobHandleStub)
+ testScalaHandle onJobQueued {
+ lock.countDown()
+ }
+ ScalaClientTestUtils.assertAwait(lock)
+ }
+
+ test("onJobStarted") {
+ val jobHandleStub = new AbstractJobHandleStub[String] {
+ override def addListener(l: Listener[String]): Unit = l.onJobStarted(this)
+ }
+ val lock = new CountDownLatch(1)
+ val testScalaHandle = new ScalaJobHandle(jobHandleStub)
+ testScalaHandle onJobStarted {
+ lock.countDown()
+ }
+ ScalaClientTestUtils.assertAwait(lock)
+ }
+}
+
+private abstract class AbstractJobHandleStub[T] private[livy] extends JobHandle[T] {
+
+ override def getState: State = null
+
+ override def addListener(l: Listener[T]): Unit = {}
+
+ override def isCancelled: Boolean = false
+
+ override def get(): T = null.asInstanceOf[T]
+
+ override def get(timeout: Long, unit: TimeUnit): T = null.asInstanceOf[T]
+
+ override def cancel(mayInterruptIfRunning: Boolean): Boolean = false
+
+ override def isDone: Boolean = true
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala/pom.xml
----------------------------------------------------------------------
diff --git a/scala/pom.xml b/scala/pom.xml
index 8a2d6ee..d304bb3 100644
--- a/scala/pom.xml
+++ b/scala/pom.xml
@@ -19,15 +19,15 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>multi-scala-project-root</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scalastyle.xml
----------------------------------------------------------------------
diff --git a/scalastyle.xml b/scalastyle.xml
index 2b06267..28609fb 100644
--- a/scalastyle.xml
+++ b/scalastyle.xml
@@ -122,8 +122,8 @@
<parameter name="groups">java,scala,3rdParty,livy</parameter>
<parameter name="group.java">javax?\..*</parameter>
<parameter name="group.scala">scala\..*</parameter>
- <parameter name="group.3rdParty">(?!com\.cloudera.livy\.).*</parameter>
- <parameter name="group.livy">com\.cloudera\.livy\..*</parameter>
+ <parameter name="group.3rdParty">(?!org\.apache.livy\.).*</parameter>
+ <parameter name="group.livy">org\.apache\.livy\..*</parameter>
</parameters>
</check>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 280f7c5..1ad6bfa 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -20,14 +20,14 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>livy-server</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<dependencies>
@@ -53,7 +53,7 @@
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-test-lib</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -171,7 +171,7 @@
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-repl_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -234,7 +234,7 @@
<configuration>
<archive>
<manifest>
- <mainClass>com.cloudera.livy.server.Main</mainClass>
+ <mainClass>org.apache.livy.server.Main</mainClass>
</manifest>
</archive>
<outputDirectory>${project.build.directory}/jars</outputDirectory>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/all-sessions.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/all-sessions.js b/server/src/main/resources/com/cloudera/livy/server/ui/static/all-sessions.js
deleted file mode 100644
index 4fe3f8f..0000000
--- a/server/src/main/resources/com/cloudera/livy/server/ui/static/all-sessions.js
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-function appIdLink(session) {
- var appUiUrl = session.appInfo.sparkUiUrl;
- if (appUiUrl != null) {
- return '<a href="' + appUiUrl + '">' + session.appId + "</a>";
- } else {
- return session.appId;
- }
-}
-
-function tdWrap(str) {
- return "<td>" + str + "</td>";
-}
-
-function loadSessionsTable(sessions) {
- $.each(sessions, function(index, session) {
- $("#interactive-sessions .sessions-table-body").append(
- "<tr>" +
- tdWrap(session.id) +
- tdWrap(appIdLink(session)) +
- tdWrap(session.owner) +
- tdWrap(session.proxyUser) +
- tdWrap(session.kind) +
- tdWrap(session.state) +
- "</tr>"
- );
- });
-}
-
-function loadBatchesTable(sessions) {
- $.each(sessions, function(index, session) {
- $("#batches .sessions-table-body").append(
- "<tr>" +
- tdWrap(session.id) +
- tdWrap(appIdLink(session)) +
- tdWrap(session.state) +
- "</tr>"
- );
- });
-}
-
-var numSessions = 0;
-var numBatches = 0;
-
-$(document).ready(function () {
- $.extend( $.fn.dataTable.defaults, {
- stateSave: true,
- });
-
- var sessionsReq = $.getJSON(location.origin + "/sessions", function(response) {
- if (response && response.total > 0) {
- $("#interactive-sessions").load("/static/sessions-table.html .sessions-template", function() {
- loadSessionsTable(response.sessions);
- $("#interactive-sessions-table").DataTable();
- $('#interactive-sessions [data-toggle="tooltip"]').tooltip();
- });
- }
- numSessions = response.total;
- });
-
- var batchesReq = $.getJSON(location.origin + "/batches", function(response) {
- if (response && response.total > 0) {
- $("#batches").load("/static/batches-table.html .sessions-template", function() {
- loadBatchesTable(response.sessions);
- $("#batches-table").DataTable();
- $('#batches [data-toggle="tooltip"]').tooltip();
- });
- }
- numBatches = response.total;
- });
-
- $.when(sessionsReq, batchesReq).done(function () {
- if (numSessions + numBatches == 0) {
- $("#all-sessions").append('<h4>No Sessions or Batches have been created yet.</h4>');
- }
- });
-});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/batches-table.html
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/batches-table.html b/server/src/main/resources/com/cloudera/livy/server/ui/static/batches-table.html
deleted file mode 100644
index e0b3213..0000000
--- a/server/src/main/resources/com/cloudera/livy/server/ui/static/batches-table.html
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<h4 id="batches-header" class="sessions-template">Batch Sessions</h4>
-
-<table id="batches-table" class="table table-striped sessions-table sessions-template">
- <thead class="sessions-table-head">
- <tr>
- <th>Batch Id</th>
- <th>
- <span data-toggle="tooltip"
- title="Spark Application Id for this session.
- If available, links to Spark Application Web UI">
- Application Id
- </span>
- </th>
- <th>
- <span data-toggle="tooltip"
- title="Session State (not_started, starting, idle, busy,
- shutting_down, error, dead, success)">
- State
- </span>
- </th>
- </tr>
- </thead>
- <tbody class="sessions-table-body">
- </tbody>
-</table>
\ No newline at end of file
[03/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/utils/SparkYarnAppSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/utils/SparkYarnAppSpec.scala b/server/src/test/scala/com/cloudera/livy/utils/SparkYarnAppSpec.scala
deleted file mode 100644
index 37f001a..0000000
--- a/server/src/test/scala/com/cloudera/livy/utils/SparkYarnAppSpec.scala
+++ /dev/null
@@ -1,353 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.utils
-
-import java.util.concurrent.{CountDownLatch, TimeUnit}
-
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import org.apache.hadoop.yarn.api.records._
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus.UNDEFINED
-import org.apache.hadoop.yarn.api.records.YarnApplicationState._
-import org.apache.hadoop.yarn.client.api.YarnClient
-import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException
-import org.apache.hadoop.yarn.util.ConverterUtils
-import org.mockito.Mockito._
-import org.mockito.invocation.InvocationOnMock
-import org.mockito.stubbing.Answer
-import org.scalatest.FunSpec
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf}
-import com.cloudera.livy.util.LineBufferedProcess
-import com.cloudera.livy.utils.SparkApp._
-
-class SparkYarnAppSpec extends FunSpec with LivyBaseUnitTestSuite {
- private def cleanupThread(t: Thread)(f: => Unit) = {
- try { f } finally { t.interrupt() }
- }
-
- private def mockSleep(ms: Long) = {
- Thread.`yield`()
- }
-
- describe("SparkYarnApp") {
- val TEST_TIMEOUT = 30 seconds
- val appId = ConverterUtils.toApplicationId("application_1467912463905_0021")
- val appIdOption = Some(appId.toString)
- val appTag = "fakeTag"
- val livyConf = new LivyConf()
- livyConf.set(LivyConf.YARN_APP_LOOKUP_TIMEOUT, "30s")
-
- it("should poll YARN state and terminate") {
- Clock.withSleepMethod(mockSleep) {
- val mockYarnClient = mock[YarnClient]
- val mockAppListener = mock[SparkAppListener]
-
- val mockAppReport = mock[ApplicationReport]
- when(mockAppReport.getApplicationId).thenReturn(appId)
- when(mockAppReport.getFinalApplicationStatus).thenReturn(FinalApplicationStatus.SUCCEEDED)
- // Simulate YARN app state progression.
- when(mockAppReport.getYarnApplicationState).thenAnswer(new Answer[YarnApplicationState]() {
- private var stateSeq = List(ACCEPTED, RUNNING, FINISHED)
-
- override def answer(invocation: InvocationOnMock): YarnApplicationState = {
- val currentState = stateSeq.head
- if (stateSeq.tail.nonEmpty) {
- stateSeq = stateSeq.tail
- }
- currentState
- }
- })
- when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
-
- val app = new SparkYarnApp(
- appTag,
- appIdOption,
- None,
- Some(mockAppListener),
- livyConf,
- mockYarnClient)
- cleanupThread(app.yarnAppMonitorThread) {
- app.yarnAppMonitorThread.join(TEST_TIMEOUT.toMillis)
- assert(!app.yarnAppMonitorThread.isAlive,
- "YarnAppMonitorThread should terminate after YARN app is finished.")
- verify(mockYarnClient, atLeast(1)).getApplicationReport(appId)
- verify(mockAppListener).stateChanged(State.STARTING, State.RUNNING)
- verify(mockAppListener).stateChanged(State.RUNNING, State.FINISHED)
- }
- }
- }
-
- it("should kill yarn app") {
- Clock.withSleepMethod(mockSleep) {
- val diag = "DIAG"
- val mockYarnClient = mock[YarnClient]
-
- val mockAppReport = mock[ApplicationReport]
- when(mockAppReport.getApplicationId).thenReturn(appId)
- when(mockAppReport.getDiagnostics).thenReturn(diag)
- when(mockAppReport.getFinalApplicationStatus).thenReturn(FinalApplicationStatus.SUCCEEDED)
-
- var appKilled = false
- when(mockAppReport.getYarnApplicationState).thenAnswer(new Answer[YarnApplicationState]() {
- override def answer(invocation: InvocationOnMock): YarnApplicationState = {
- if (!appKilled) {
- RUNNING
- } else {
- KILLED
- }
- }
- })
- when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
-
- val app = new SparkYarnApp(appTag, appIdOption, None, None, livyConf, mockYarnClient)
- cleanupThread(app.yarnAppMonitorThread) {
- app.kill()
- appKilled = true
-
- app.yarnAppMonitorThread.join(TEST_TIMEOUT.toMillis)
- assert(!app.yarnAppMonitorThread.isAlive,
- "YarnAppMonitorThread should terminate after YARN app is finished.")
- verify(mockYarnClient, atLeast(1)).getApplicationReport(appId)
- verify(mockYarnClient).killApplication(appId)
- assert(app.log().mkString.contains(diag))
- }
- }
- }
-
- it("should return spark-submit log") {
- Clock.withSleepMethod(mockSleep) {
- val mockYarnClient = mock[YarnClient]
- val mockSparkSubmit = mock[LineBufferedProcess]
- val sparkSubmitInfoLog = IndexedSeq("SPARK-SUBMIT", "LOG")
- val sparkSubmitErrorLog = IndexedSeq("SPARK-SUBMIT", "error log")
- val sparkSubmitLog = ("stdout: " +: sparkSubmitInfoLog) ++
- ("\nstderr: " +: sparkSubmitErrorLog) :+ "\nYARN Diagnostics: "
- when(mockSparkSubmit.inputLines).thenReturn(sparkSubmitInfoLog)
- when(mockSparkSubmit.errorLines).thenReturn(sparkSubmitErrorLog)
- val waitForCalledLatch = new CountDownLatch(1)
- when(mockSparkSubmit.waitFor()).thenAnswer(new Answer[Int]() {
- override def answer(invocation: InvocationOnMock): Int = {
- waitForCalledLatch.countDown()
- 0
- }
- })
-
- val mockAppReport = mock[ApplicationReport]
- when(mockAppReport.getApplicationId).thenReturn(appId)
- when(mockAppReport.getYarnApplicationState).thenReturn(YarnApplicationState.FINISHED)
- when(mockAppReport.getDiagnostics).thenReturn(null)
- when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
-
- val app = new SparkYarnApp(
- appTag,
- appIdOption,
- Some(mockSparkSubmit),
- None,
- livyConf,
- mockYarnClient)
- cleanupThread(app.yarnAppMonitorThread) {
- waitForCalledLatch.await(TEST_TIMEOUT.toMillis, TimeUnit.MILLISECONDS)
- assert(app.log() == sparkSubmitLog, "Expect spark-submit log")
- }
- }
- }
-
- it("can kill spark-submit while it's running") {
- Clock.withSleepMethod(mockSleep) {
- val livyConf = new LivyConf()
- livyConf.set(LivyConf.YARN_APP_LOOKUP_TIMEOUT, "0")
-
- val mockYarnClient = mock[YarnClient]
- val mockSparkSubmit = mock[LineBufferedProcess]
-
- val sparkSubmitRunningLatch = new CountDownLatch(1)
- // Simulate a running spark-submit
- when(mockSparkSubmit.waitFor()).thenAnswer(new Answer[Int]() {
- override def answer(invocation: InvocationOnMock): Int = {
- sparkSubmitRunningLatch.await()
- 0
- }
- })
-
- val app = new SparkYarnApp(
- appTag,
- appIdOption,
- Some(mockSparkSubmit),
- None,
- livyConf,
- mockYarnClient)
- cleanupThread(app.yarnAppMonitorThread) {
- app.kill()
- verify(mockSparkSubmit, times(1)).destroy()
- sparkSubmitRunningLatch.countDown()
- }
- }
- }
-
- it("should map YARN state to SparkApp.State correctly") {
- val app = new SparkYarnApp(appTag, appIdOption, None, None, livyConf)
- cleanupThread(app.yarnAppMonitorThread) {
- assert(app.mapYarnState(appId, NEW, UNDEFINED) == State.STARTING)
- assert(app.mapYarnState(appId, NEW_SAVING, UNDEFINED) == State.STARTING)
- assert(app.mapYarnState(appId, SUBMITTED, UNDEFINED) == State.STARTING)
- assert(app.mapYarnState(appId, ACCEPTED, UNDEFINED) == State.STARTING)
- assert(app.mapYarnState(appId, RUNNING, UNDEFINED) == State.RUNNING)
- assert(
- app.mapYarnState(appId, FINISHED, FinalApplicationStatus.SUCCEEDED) == State.FINISHED)
- assert(app.mapYarnState(appId, FINISHED, FinalApplicationStatus.FAILED) == State.FAILED)
- assert(app.mapYarnState(appId, FINISHED, FinalApplicationStatus.KILLED) == State.KILLED)
- assert(app.mapYarnState(appId, FINISHED, UNDEFINED) == State.FAILED)
- assert(app.mapYarnState(appId, FAILED, UNDEFINED) == State.FAILED)
- assert(app.mapYarnState(appId, KILLED, UNDEFINED) == State.KILLED)
- }
- }
-
- it("should expose driver log url and Spark UI url") {
- Clock.withSleepMethod(mockSleep) {
- val mockYarnClient = mock[YarnClient]
- val driverLogUrl = "DRIVER LOG URL"
- val sparkUiUrl = "SPARK UI URL"
-
- val mockApplicationAttemptId = mock[ApplicationAttemptId]
- val mockAppReport = mock[ApplicationReport]
- when(mockAppReport.getApplicationId).thenReturn(appId)
- when(mockAppReport.getFinalApplicationStatus).thenReturn(FinalApplicationStatus.SUCCEEDED)
- when(mockAppReport.getTrackingUrl).thenReturn(sparkUiUrl)
- when(mockAppReport.getCurrentApplicationAttemptId).thenReturn(mockApplicationAttemptId)
- var done = false
- when(mockAppReport.getYarnApplicationState).thenAnswer(new Answer[YarnApplicationState]() {
- override def answer(invocation: InvocationOnMock): YarnApplicationState = {
- if (!done) {
- RUNNING
- } else {
- FINISHED
- }
- }
- })
- when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
-
- val mockAttemptReport = mock[ApplicationAttemptReport]
- val mockContainerId = mock[ContainerId]
- when(mockAttemptReport.getAMContainerId).thenReturn(mockContainerId)
- when(mockYarnClient.getApplicationAttemptReport(mockApplicationAttemptId))
- .thenReturn(mockAttemptReport)
-
- val mockContainerReport = mock[ContainerReport]
- when(mockYarnClient.getContainerReport(mockContainerId)).thenReturn(mockContainerReport)
-
- // Block test until getLogUrl is called 10 times.
- val getLogUrlCountDown = new CountDownLatch(10)
- when(mockContainerReport.getLogUrl).thenAnswer(new Answer[String] {
- override def answer(invocation: InvocationOnMock): String = {
- getLogUrlCountDown.countDown()
- driverLogUrl
- }
- })
-
- val mockListener = mock[SparkAppListener]
-
- val app = new SparkYarnApp(
- appTag, appIdOption, None, Some(mockListener), livyConf, mockYarnClient)
- cleanupThread(app.yarnAppMonitorThread) {
- getLogUrlCountDown.await(TEST_TIMEOUT.length, TEST_TIMEOUT.unit)
- done = true
-
- app.yarnAppMonitorThread.join(TEST_TIMEOUT.toMillis)
- assert(!app.yarnAppMonitorThread.isAlive,
- "YarnAppMonitorThread should terminate after YARN app is finished.")
-
- verify(mockYarnClient, atLeast(1)).getApplicationReport(appId)
- verify(mockAppReport, atLeast(1)).getTrackingUrl()
- verify(mockContainerReport, atLeast(1)).getLogUrl()
- verify(mockListener).appIdKnown(appId.toString)
- verify(mockListener).infoChanged(AppInfo(Some(driverLogUrl), Some(sparkUiUrl)))
- }
- }
- }
-
- it("should not die on YARN-4411") {
- Clock.withSleepMethod(mockSleep) {
- val mockYarnClient = mock[YarnClient]
-
- // Block test until getApplicationReport is called 10 times.
- val pollCountDown = new CountDownLatch(10)
- when(mockYarnClient.getApplicationReport(appId)).thenAnswer(new Answer[ApplicationReport] {
- override def answer(invocation: InvocationOnMock): ApplicationReport = {
- pollCountDown.countDown()
- throw new IllegalArgumentException("No enum constant " +
- "org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState.FINAL_SAVING")
- }
- })
-
- val app = new SparkYarnApp(appTag, appIdOption, None, None, livyConf, mockYarnClient)
- cleanupThread(app.yarnAppMonitorThread) {
- pollCountDown.await(TEST_TIMEOUT.length, TEST_TIMEOUT.unit)
- assert(app.state == SparkApp.State.STARTING)
-
- app.state = SparkApp.State.FINISHED
- app.yarnAppMonitorThread.join(TEST_TIMEOUT.toMillis)
- }
- }
- }
-
- it("should not die on ApplicationAttemptNotFoundException") {
- Clock.withSleepMethod(mockSleep) {
- val mockYarnClient = mock[YarnClient]
- val mockAppReport = mock[ApplicationReport]
- val mockApplicationAttemptId = mock[ApplicationAttemptId]
- var done = false
-
- when(mockAppReport.getApplicationId).thenReturn(appId)
- when(mockAppReport.getYarnApplicationState).thenAnswer(
- new Answer[YarnApplicationState]() {
- override def answer(invocation: InvocationOnMock): YarnApplicationState = {
- if (done) {
- FINISHED
- } else {
- RUNNING
- }
- }
- })
- when(mockAppReport.getFinalApplicationStatus).thenReturn(FinalApplicationStatus.SUCCEEDED)
- when(mockAppReport.getCurrentApplicationAttemptId).thenReturn(mockApplicationAttemptId)
- when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
-
- // Block test until getApplicationReport is called 10 times.
- val pollCountDown = new CountDownLatch(10)
- when(mockYarnClient.getApplicationAttemptReport(mockApplicationAttemptId)).thenAnswer(
- new Answer[ApplicationReport] {
- override def answer(invocation: InvocationOnMock): ApplicationReport = {
- pollCountDown.countDown()
- throw new ApplicationAttemptNotFoundException("unit test")
- }
- })
-
- val app = new SparkYarnApp(appTag, appIdOption, None, None, livyConf, mockYarnClient)
- cleanupThread(app.yarnAppMonitorThread) {
- pollCountDown.await(TEST_TIMEOUT.length, TEST_TIMEOUT.unit)
- assert(app.state == SparkApp.State.RUNNING)
- done = true
-
- app.yarnAppMonitorThread.join(TEST_TIMEOUT.toMillis)
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/ApiVersioningSupportSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/ApiVersioningSupportSpec.scala b/server/src/test/scala/org/apache/livy/server/ApiVersioningSupportSpec.scala
new file mode 100644
index 0000000..0f50ced
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/ApiVersioningSupportSpec.scala
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import javax.servlet.http.HttpServletResponse
+
+import org.scalatest.FunSpecLike
+import org.scalatra.ScalatraServlet
+import org.scalatra.test.scalatest.ScalatraSuite
+
+import org.apache.livy.LivyBaseUnitTestSuite
+
+class ApiVersioningSupportSpec extends ScalatraSuite with FunSpecLike with LivyBaseUnitTestSuite {
+ val LatestVersionOutput = "latest"
+
+ object FakeApiVersions extends Enumeration {
+ type FakeApiVersions = Value
+ val v0_1 = Value("0.1")
+ val v0_2 = Value("0.2")
+ val v1_0 = Value("1.0")
+ }
+
+ import FakeApiVersions._
+
+ class MockServlet extends ScalatraServlet with AbstractApiVersioningSupport {
+ override val apiVersions = FakeApiVersions
+ override type ApiVersionType = FakeApiVersions.Value
+
+ get("/test") {
+ response.writer.write(LatestVersionOutput)
+ }
+
+ get("/test", apiVersion <= v0_2) {
+ response.writer.write(v0_2.toString)
+ }
+
+ get("/test", apiVersion <= v0_1) {
+ response.writer.write(v0_1.toString)
+ }
+
+ get("/droppedApi", apiVersion <= v0_2) {
+ }
+
+ get("/newApi", apiVersion >= v0_2) {
+ }
+ }
+
+ var mockServlet: MockServlet = new MockServlet
+ addServlet(mockServlet, "/*")
+
+ def generateHeader(acceptHeader: String): Map[String, String] = {
+ if (acceptHeader != null) Map("Accept" -> acceptHeader) else Map.empty
+ }
+
+ def shouldReturn(url: String, acceptHeader: String, expectedVersion: String = null): Unit = {
+ get(url, headers = generateHeader(acceptHeader)) {
+ status should equal(200)
+ if (expectedVersion != null) {
+ body should equal(expectedVersion)
+ }
+ }
+ }
+
+ def shouldFail(url: String, acceptHeader: String, expectedErrorCode: Int): Unit = {
+ get(url, headers = generateHeader(acceptHeader)) {
+ status should equal(expectedErrorCode)
+ }
+ }
+
+ it("should pick the latest API version if Accept header is unspecified") {
+ shouldReturn("/test", null, LatestVersionOutput)
+ }
+
+ it("should pick the latest API version if Accept header does not specify any version") {
+ shouldReturn("/test", "foo", LatestVersionOutput)
+ shouldReturn("/test", "application/vnd.random.v1.1", LatestVersionOutput)
+ shouldReturn("/test", "application/vnd.livy.+json", LatestVersionOutput)
+ }
+
+ it("should pick the correct API version") {
+ shouldReturn("/test", "application/vnd.livy.v0.1", v0_1.toString)
+ shouldReturn("/test", "application/vnd.livy.v0.2+", v0_2.toString)
+ shouldReturn("/test", "application/vnd.livy.v0.1+bar", v0_1.toString)
+ shouldReturn("/test", "application/vnd.livy.v0.2+foo", v0_2.toString)
+ shouldReturn("/test", "application/vnd.livy.v0.1+vnd.livy.v0.2", v0_1.toString)
+ shouldReturn("/test", "application/vnd.livy.v0.2++++++++++++++++", v0_2.toString)
+ shouldReturn("/test", "application/vnd.livy.v1.0", LatestVersionOutput)
+ }
+
+ it("should return error when the specified API version does not exist") {
+ shouldFail("/test", "application/vnd.livy.v", HttpServletResponse.SC_NOT_ACCEPTABLE)
+ shouldFail("/test", "application/vnd.livy.v+json", HttpServletResponse.SC_NOT_ACCEPTABLE)
+ shouldFail("/test", "application/vnd.livy.v666.666", HttpServletResponse.SC_NOT_ACCEPTABLE)
+ shouldFail("/test", "application/vnd.livy.v666.666+json", HttpServletResponse.SC_NOT_ACCEPTABLE)
+ shouldFail("/test", "application/vnd.livy.v1.1+json", HttpServletResponse.SC_NOT_ACCEPTABLE)
+ }
+
+ it("should not see a dropped API") {
+ shouldReturn("/droppedApi", "application/vnd.livy.v0.1+json")
+ shouldReturn("/droppedApi", "application/vnd.livy.v0.2+json")
+ shouldFail("/droppedApi", "application/vnd.livy.v1.0+json", HttpServletResponse.SC_NOT_FOUND)
+ }
+
+ it("should not see a new API at an older version") {
+ shouldFail("/newApi", "application/vnd.livy.v0.1+json", HttpServletResponse.SC_NOT_FOUND)
+ shouldReturn("/newApi", "application/vnd.livy.v0.2+json")
+ shouldReturn("/newApi", "application/vnd.livy.v1.0+json")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/BaseJsonServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/BaseJsonServletSpec.scala b/server/src/test/scala/org/apache/livy/server/BaseJsonServletSpec.scala
new file mode 100644
index 0000000..959707a
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/BaseJsonServletSpec.scala
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import java.io.ByteArrayOutputStream
+import javax.servlet.http.HttpServletResponse._
+
+import scala.reflect.ClassTag
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import org.scalatest.FunSpecLike
+import org.scalatra.test.scalatest.ScalatraSuite
+
+import org.apache.livy.LivyBaseUnitTestSuite
+
+/**
+ * Base class that enhances ScalatraSuite so that it's easier to test JsonServlet
+ * implementations. Variants of the test methods (get, post, etc) exist with the "j"
+ * prefix; these automatically serialize the body of the request to JSON, and
+ * deserialize the result from JSON.
+ *
+ * In case the response is not JSON, the expected type for the test function should be
+ * `Unit`, and the `response` object should be checked directly.
+ */
+abstract class BaseJsonServletSpec extends ScalatraSuite
+ with FunSpecLike with LivyBaseUnitTestSuite {
+
+ protected val mapper = new ObjectMapper()
+ .registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
+
+ protected val defaultHeaders: Map[String, String] = Map("Content-Type" -> "application/json")
+
+ protected def jdelete[R: ClassTag](
+ uri: String,
+ expectedStatus: Int = SC_OK,
+ headers: Map[String, String] = defaultHeaders)
+ (fn: R => Unit): Unit = {
+ delete(uri, headers = headers)(doTest(expectedStatus, fn))
+ }
+
+ protected def jget[R: ClassTag](
+ uri: String,
+ expectedStatus: Int = SC_OK,
+ headers: Map[String, String] = defaultHeaders)
+ (fn: R => Unit): Unit = {
+ get(uri, headers = headers)(doTest(expectedStatus, fn))
+ }
+
+ protected def jpatch[R: ClassTag](
+ uri: String,
+ body: AnyRef,
+ expectedStatus: Int = SC_OK,
+ headers: Map[String, String] = defaultHeaders)
+ (fn: R => Unit): Unit = {
+ patch(uri, body = toJson(body), headers = headers)(doTest(expectedStatus, fn))
+ }
+
+ protected def jpost[R: ClassTag](
+ uri: String,
+ body: AnyRef,
+ expectedStatus: Int = SC_CREATED,
+ headers: Map[String, String] = defaultHeaders)
+ (fn: R => Unit): Unit = {
+ post(uri, body = toJson(body), headers = headers)(doTest(expectedStatus, fn))
+ }
+
+ /** A version of jpost specific for testing file upload. */
+ protected def jupload[R: ClassTag](
+ uri: String,
+ files: Iterable[(String, Any)],
+ headers: Map[String, String] = Map(),
+ expectedStatus: Int = SC_OK)
+ (fn: R => Unit): Unit = {
+ post(uri, Map.empty, files)(doTest(expectedStatus, fn))
+ }
+
+ protected def jput[R: ClassTag](
+ uri: String,
+ body: AnyRef,
+ expectedStatus: Int = SC_OK,
+ headers: Map[String, String] = defaultHeaders)
+ (fn: R => Unit): Unit = {
+ put(uri, body = toJson(body), headers = headers)(doTest(expectedStatus, fn))
+ }
+
+ private def doTest[R: ClassTag](expectedStatus: Int, fn: R => Unit)
+ (implicit klass: ClassTag[R]): Unit = {
+ if (status != expectedStatus) {
+ // Yeah this is weird, but we don't want to evaluate "response.body" if there's no error.
+ assert(status === expectedStatus,
+ s"Unexpected response status: $status != $expectedStatus (${response.body})")
+ }
+ // Only try to parse the body if response is in the "OK" range (20x).
+ if ((status / 100) * 100 == SC_OK) {
+ val result =
+ if (header("Content-Type").startsWith("application/json")) {
+ // Sometimes there's an empty body with no "Content-Length" header. So read the whole
+ // body first, and only send it to Jackson if there's content.
+ val in = response.inputStream
+ val out = new ByteArrayOutputStream()
+ val buf = new Array[Byte](1024)
+ var read = 0
+ while (read >= 0) {
+ read = in.read(buf)
+ if (read > 0) {
+ out.write(buf, 0, read)
+ }
+ }
+
+ val data = out.toByteArray()
+ if (data.length > 0) {
+ mapper.readValue(data, klass.runtimeClass)
+ } else {
+ null
+ }
+ } else {
+ assert(klass.runtimeClass == classOf[Unit])
+ ()
+ }
+ fn(result.asInstanceOf[R])
+ }
+ }
+
+ private def toJson(obj: Any): Array[Byte] = mapper.writeValueAsBytes(obj)
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/BaseSessionServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/BaseSessionServletSpec.scala b/server/src/test/scala/org/apache/livy/server/BaseSessionServletSpec.scala
new file mode 100644
index 0000000..203f1f7
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/BaseSessionServletSpec.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import javax.servlet.http.HttpServletRequest
+
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.livy.LivyConf
+import org.apache.livy.sessions.Session
+import org.apache.livy.sessions.Session.RecoveryMetadata
+
+object BaseSessionServletSpec {
+
+ /** Header used to override the user remote user in tests. */
+ val REMOTE_USER_HEADER = "X-Livy-SessionServlet-User"
+
+}
+
+abstract class BaseSessionServletSpec[S <: Session, R <: RecoveryMetadata]
+ extends BaseJsonServletSpec
+ with BeforeAndAfterAll {
+
+ /** Config map containing option that is blacklisted. */
+ protected val BLACKLISTED_CONFIG = Map("spark.do_not_set" -> "true")
+
+ /** Name of the admin user. */
+ protected val ADMIN = "__admin__"
+
+ /** Create headers that identify a specific user in tests. */
+ protected def makeUserHeaders(user: String): Map[String, String] = {
+ defaultHeaders ++ Map(BaseSessionServletSpec.REMOTE_USER_HEADER -> user)
+ }
+
+ protected val adminHeaders = makeUserHeaders(ADMIN)
+
+ /** Create a LivyConf with impersonation enabled and a superuser. */
+ protected def createConf(): LivyConf = {
+ new LivyConf()
+ .set(LivyConf.IMPERSONATION_ENABLED, true)
+ .set(LivyConf.SUPERUSERS, ADMIN)
+ .set(LivyConf.LOCAL_FS_WHITELIST, sys.props("java.io.tmpdir"))
+ }
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ servlet.shutdown()
+ }
+
+ def createServlet(): SessionServlet[S, R]
+
+ protected val servlet = createServlet()
+
+ addServlet(servlet, "/*")
+
+ protected def toJson(msg: AnyRef): Array[Byte] = mapper.writeValueAsBytes(msg)
+
+}
+
+trait RemoteUserOverride {
+ this: SessionServlet[_, _] =>
+
+ override protected def remoteUser(req: HttpServletRequest): String = {
+ req.getHeader(BaseSessionServletSpec.REMOTE_USER_HEADER)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/JsonServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/JsonServletSpec.scala b/server/src/test/scala/org/apache/livy/server/JsonServletSpec.scala
new file mode 100644
index 0000000..5ca3997
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/JsonServletSpec.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import java.nio.charset.StandardCharsets.UTF_8
+import javax.servlet.http.HttpServletResponse._
+
+import org.scalatra._
+
+class JsonServletSpec extends BaseJsonServletSpec {
+
+ addServlet(new TestJsonServlet(), "/*")
+
+ describe("The JSON servlet") {
+
+ it("should serialize result of delete") {
+ jdelete[MethodReturn]("/delete") { result =>
+ assert(result.value === "delete")
+ }
+ }
+
+ it("should serialize result of get") {
+ jget[MethodReturn]("/get") { result =>
+ assert(result.value === "get")
+ }
+ }
+
+ it("should serialize an ActionResult's body") {
+ jpost[MethodReturn]("/post", MethodArg("post")) { result =>
+ assert(result.value === "post")
+ }
+ }
+
+ it("should wrap a raw result") {
+ jput[MethodReturn]("/put", MethodArg("put")) { result =>
+ assert(result.value === "put")
+ }
+ }
+
+ it("should bypass non-json results") {
+ jpatch[Unit]("/patch", MethodArg("patch"), expectedStatus = SC_NOT_FOUND) { _ =>
+ assert(response.body === "patch")
+ }
+ }
+
+ it("should translate JSON errors to BadRequest") {
+ post("/post", "abcde".getBytes(UTF_8), headers = defaultHeaders) {
+ assert(status === SC_BAD_REQUEST)
+ }
+ }
+
+ it("should translate bad param name to BadRequest") {
+ post("/post", """{"value1":"1"}""".getBytes(UTF_8), headers = defaultHeaders) {
+ assert(status === SC_BAD_REQUEST)
+ }
+ }
+
+ it("should translate type mismatch to BadRequest") {
+ post("/postlist", """{"listParam":"1"}""".getBytes(UTF_8), headers = defaultHeaders) {
+ assert(status === SC_BAD_REQUEST)
+ }
+ }
+
+ it("should respect user-installed error handlers") {
+ post("/error", headers = defaultHeaders) {
+ assert(status === SC_SERVICE_UNAVAILABLE)
+ assert(response.body === "error")
+ }
+ }
+
+ it("should handle empty return values") {
+ jget[MethodReturn]("/empty") { result =>
+ assert(result == null)
+ }
+ }
+
+ }
+
+}
+
+private case class MethodArg(value: String)
+
+private case class MethodReturn(value: String)
+
+private case class MethodReturnList(listParam: List[String] = List())
+
+private class TestJsonServlet extends JsonServlet {
+
+ before() {
+ contentType = "application/json"
+ }
+
+ delete("/delete") {
+ Ok(MethodReturn("delete"))
+ }
+
+ get("/get") {
+ Ok(MethodReturn("get"))
+ }
+
+ jpost[MethodArg]("/post") { arg =>
+ Created(MethodReturn(arg.value))
+ }
+
+ jpost[MethodReturnList]("/postlist") { arg =>
+ Created()
+ }
+
+ jput[MethodArg]("/put") { arg =>
+ MethodReturn(arg.value)
+ }
+
+ jpatch[MethodArg]("/patch") { arg =>
+ contentType = "text/plain"
+ NotFound(arg.value)
+ }
+
+ get("/empty") {
+ ()
+ }
+
+ post("/error") {
+ throw new IllegalStateException("error")
+ }
+
+ // Install an error handler to make sure the parent's still work.
+ error {
+ case e: IllegalStateException =>
+ contentType = "text/plain"
+ ServiceUnavailable(e.getMessage())
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/SessionServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/SessionServletSpec.scala b/server/src/test/scala/org/apache/livy/server/SessionServletSpec.scala
new file mode 100644
index 0000000..292a9cd
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/SessionServletSpec.scala
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server
+
+import javax.servlet.http.HttpServletRequest
+import javax.servlet.http.HttpServletResponse._
+
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.LivyConf
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.{Session, SessionManager, SessionState}
+import org.apache.livy.sessions.Session.RecoveryMetadata
+
+object SessionServletSpec {
+
+ val PROXY_USER = "proxyUser"
+
+ class MockSession(id: Int, owner: String, livyConf: LivyConf)
+ extends Session(id, owner, livyConf) {
+
+ case class MockRecoveryMetadata(id: Int) extends RecoveryMetadata()
+
+ override val proxyUser = None
+
+ override def recoveryMetadata: RecoveryMetadata = MockRecoveryMetadata(0)
+
+ override def state: SessionState = SessionState.Idle()
+
+ override protected def stopSession(): Unit = ()
+
+ override def logLines(): IndexedSeq[String] = IndexedSeq("log")
+
+ }
+
+ case class MockSessionView(id: Int, owner: String, logs: Seq[String])
+
+}
+
+class SessionServletSpec
+ extends BaseSessionServletSpec[Session, RecoveryMetadata] {
+
+ import SessionServletSpec._
+
+ override def createServlet(): SessionServlet[Session, RecoveryMetadata] = {
+ val conf = createConf()
+ val sessionManager = new SessionManager[Session, RecoveryMetadata](
+ conf,
+ { _ => assert(false).asInstanceOf[Session] },
+ mock[SessionStore],
+ "test",
+ Some(Seq.empty))
+
+ new SessionServlet(sessionManager, conf) with RemoteUserOverride {
+ override protected def createSession(req: HttpServletRequest): Session = {
+ val params = bodyAs[Map[String, String]](req)
+ checkImpersonation(params.get(PROXY_USER), req)
+ new MockSession(sessionManager.nextId(), remoteUser(req), conf)
+ }
+
+ override protected def clientSessionView(
+ session: Session,
+ req: HttpServletRequest): Any = {
+ val logs = if (hasAccess(session.owner, req)) session.logLines() else Nil
+ MockSessionView(session.id, session.owner, logs)
+ }
+ }
+ }
+
+ private val aliceHeaders = makeUserHeaders("alice")
+ private val bobHeaders = makeUserHeaders("bob")
+
+ private def delete(id: Int, headers: Map[String, String], expectedStatus: Int): Unit = {
+ jdelete[Map[String, Any]](s"/$id", headers = headers, expectedStatus = expectedStatus) { _ =>
+ // Nothing to do.
+ }
+ }
+
+ describe("SessionServlet") {
+
+ it("should return correct Location in header") {
+ // mount to "/sessions/*" to test. If request URI is "/session", getPathInfo() will
+ // return null, since there's no extra path.
+ // mount to "/*" will always return "/", so that it cannot reflect the issue.
+ addServlet(servlet, "/sessions/*")
+ jpost[MockSessionView]("/sessions", Map(), headers = aliceHeaders) { res =>
+ assert(header("Location") === "/sessions/0")
+ jdelete[Map[String, Any]]("/sessions/0", SC_OK, aliceHeaders) { _ => }
+ }
+ }
+
+ it("should attach owner information to sessions") {
+ jpost[MockSessionView]("/", Map(), headers = aliceHeaders) { res =>
+ assert(res.owner === "alice")
+ assert(res.logs === IndexedSeq("log"))
+ delete(res.id, aliceHeaders, SC_OK)
+ }
+ }
+
+ it("should allow other users to see non-sensitive information") {
+ jpost[MockSessionView]("/", Map(), headers = aliceHeaders) { res =>
+ jget[MockSessionView](s"/${res.id}", headers = bobHeaders) { res =>
+ assert(res.owner === "alice")
+ assert(res.logs === Nil)
+ }
+ delete(res.id, aliceHeaders, SC_OK)
+ }
+ }
+
+ it("should prevent non-owners from modifying sessions") {
+ jpost[MockSessionView]("/", Map(), headers = aliceHeaders) { res =>
+ delete(res.id, bobHeaders, SC_FORBIDDEN)
+ }
+ }
+
+ it("should allow admins to access all sessions") {
+ jpost[MockSessionView]("/", Map(), headers = aliceHeaders) { res =>
+ jget[MockSessionView](s"/${res.id}", headers = adminHeaders) { res =>
+ assert(res.owner === "alice")
+ assert(res.logs === IndexedSeq("log"))
+ }
+ delete(res.id, adminHeaders, SC_OK)
+ }
+ }
+
+ it("should not allow regular users to impersonate others") {
+ jpost[MockSessionView]("/", Map(PROXY_USER -> "bob"), headers = aliceHeaders,
+ expectedStatus = SC_FORBIDDEN) { _ => }
+ }
+
+ it("should allow admins to impersonate anyone") {
+ jpost[MockSessionView]("/", Map(PROXY_USER -> "bob"), headers = adminHeaders) { res =>
+ delete(res.id, bobHeaders, SC_FORBIDDEN)
+ delete(res.id, adminHeaders, SC_OK)
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/batch/BatchServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/batch/BatchServletSpec.scala b/server/src/test/scala/org/apache/livy/server/batch/BatchServletSpec.scala
new file mode 100644
index 0000000..7f5e33d
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/batch/BatchServletSpec.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.batch
+
+import java.io.FileWriter
+import java.nio.file.{Files, Path}
+import java.util.concurrent.TimeUnit
+import javax.servlet.http.HttpServletRequest
+import javax.servlet.http.HttpServletResponse._
+
+import scala.concurrent.duration.Duration
+
+import org.mockito.Mockito._
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.Utils
+import org.apache.livy.server.BaseSessionServletSpec
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.{BatchSessionManager, SessionState}
+import org.apache.livy.utils.AppInfo
+
+class BatchServletSpec extends BaseSessionServletSpec[BatchSession, BatchRecoveryMetadata] {
+
+ val script: Path = {
+ val script = Files.createTempFile("livy-test", ".py")
+ script.toFile.deleteOnExit()
+ val writer = new FileWriter(script.toFile)
+ try {
+ writer.write(
+ """
+ |print "hello world"
+ """.stripMargin)
+ } finally {
+ writer.close()
+ }
+ script
+ }
+
+ override def createServlet(): BatchSessionServlet = {
+ val livyConf = createConf()
+ val sessionStore = mock[SessionStore]
+ new BatchSessionServlet(
+ new BatchSessionManager(livyConf, sessionStore, Some(Seq.empty)),
+ sessionStore,
+ livyConf)
+ }
+
+ describe("Batch Servlet") {
+ it("should create and tear down a batch") {
+ jget[Map[String, Any]]("/") { data =>
+ data("sessions") should equal (Seq())
+ }
+
+ val createRequest = new CreateBatchRequest()
+ createRequest.file = script.toString
+ createRequest.conf = Map("spark.driver.extraClassPath" -> sys.props("java.class.path"))
+
+ jpost[Map[String, Any]]("/", createRequest) { data =>
+ header("Location") should equal("/0")
+ data("id") should equal (0)
+
+ val batch = servlet.sessionManager.get(0)
+ batch should be (defined)
+ }
+
+ // Wait for the process to finish.
+ {
+ val batch = servlet.sessionManager.get(0).get
+ Utils.waitUntil({ () => !batch.state.isActive }, Duration(10, TimeUnit.SECONDS))
+ (batch.state match {
+ case SessionState.Success(_) => true
+ case _ => false
+ }) should be (true)
+ }
+
+ jget[Map[String, Any]]("/0") { data =>
+ data("id") should equal (0)
+ data("state") should equal ("success")
+
+ val batch = servlet.sessionManager.get(0)
+ batch should be (defined)
+ }
+
+ jget[Map[String, Any]]("/0/log?size=1000") { data =>
+ data("id") should equal (0)
+ data("log").asInstanceOf[Seq[String]] should contain ("hello world")
+
+ val batch = servlet.sessionManager.get(0)
+ batch should be (defined)
+ }
+
+ jdelete[Map[String, Any]]("/0") { data =>
+ data should equal (Map("msg" -> "deleted"))
+
+ val batch = servlet.sessionManager.get(0)
+ batch should not be defined
+ }
+ }
+
+ it("should respect config black list") {
+ val createRequest = new CreateBatchRequest()
+ createRequest.file = script.toString
+ createRequest.conf = BLACKLISTED_CONFIG
+ jpost[Map[String, Any]]("/", createRequest, expectedStatus = SC_BAD_REQUEST) { _ => }
+ }
+
+ it("should show session properties") {
+ val id = 0
+ val state = SessionState.Running()
+ val appId = "appid"
+ val appInfo = AppInfo(Some("DRIVER LOG URL"), Some("SPARK UI URL"))
+ val log = IndexedSeq[String]("log1", "log2")
+
+ val session = mock[BatchSession]
+ when(session.id).thenReturn(id)
+ when(session.state).thenReturn(state)
+ when(session.appId).thenReturn(Some(appId))
+ when(session.appInfo).thenReturn(appInfo)
+ when(session.logLines()).thenReturn(log)
+
+ val req = mock[HttpServletRequest]
+
+ val view = servlet.asInstanceOf[BatchSessionServlet].clientSessionView(session, req)
+ .asInstanceOf[BatchSessionView]
+
+ view.id shouldEqual id
+ view.state shouldEqual state.toString
+ view.appId shouldEqual Some(appId)
+ view.appInfo shouldEqual appInfo
+ view.log shouldEqual log
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/batch/BatchSessionSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/batch/BatchSessionSpec.scala b/server/src/test/scala/org/apache/livy/server/batch/BatchSessionSpec.scala
new file mode 100644
index 0000000..eb80bef
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/batch/BatchSessionSpec.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.batch
+
+import java.io.FileWriter
+import java.nio.file.{Files, Path}
+import java.util.concurrent.TimeUnit
+
+import scala.concurrent.duration.Duration
+
+import org.mockito.Matchers
+import org.mockito.Matchers.anyObject
+import org.mockito.Mockito._
+import org.scalatest.{BeforeAndAfter, FunSpec, ShouldMatchers}
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf, Utils}
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.SessionState
+import org.apache.livy.utils.{AppInfo, SparkApp}
+
+class BatchSessionSpec
+ extends FunSpec
+ with BeforeAndAfter
+ with ShouldMatchers
+ with LivyBaseUnitTestSuite {
+
+ val script: Path = {
+ val script = Files.createTempFile("livy-test", ".py")
+ script.toFile.deleteOnExit()
+ val writer = new FileWriter(script.toFile)
+ try {
+ writer.write(
+ """
+ |print "hello world"
+ """.stripMargin)
+ } finally {
+ writer.close()
+ }
+ script
+ }
+
+ describe("A Batch process") {
+ var sessionStore: SessionStore = null
+
+ before {
+ sessionStore = mock[SessionStore]
+ }
+
+ it("should create a process") {
+ val req = new CreateBatchRequest()
+ req.file = script.toString
+ req.conf = Map("spark.driver.extraClassPath" -> sys.props("java.class.path"))
+
+ val conf = new LivyConf().set(LivyConf.LOCAL_FS_WHITELIST, sys.props("java.io.tmpdir"))
+ val batch = BatchSession.create(0, req, conf, null, None, sessionStore)
+
+ Utils.waitUntil({ () => !batch.state.isActive }, Duration(10, TimeUnit.SECONDS))
+ (batch.state match {
+ case SessionState.Success(_) => true
+ case _ => false
+ }) should be (true)
+
+ batch.logLines() should contain("hello world")
+ }
+
+ it("should update appId and appInfo") {
+ val conf = new LivyConf()
+ val req = new CreateBatchRequest()
+ val mockApp = mock[SparkApp]
+ val batch = BatchSession.create(0, req, conf, null, None, sessionStore, Some(mockApp))
+
+ val expectedAppId = "APPID"
+ batch.appIdKnown(expectedAppId)
+ verify(sessionStore, atLeastOnce()).save(
+ Matchers.eq(BatchSession.RECOVERY_SESSION_TYPE), anyObject())
+ batch.appId shouldEqual Some(expectedAppId)
+
+ val expectedAppInfo = AppInfo(Some("DRIVER LOG URL"), Some("SPARK UI URL"))
+ batch.infoChanged(expectedAppInfo)
+ batch.appInfo shouldEqual expectedAppInfo
+ }
+
+ it("should recover session") {
+ val conf = new LivyConf()
+ val req = new CreateBatchRequest()
+ val mockApp = mock[SparkApp]
+ val m = BatchRecoveryMetadata(99, None, "appTag", null, None)
+ val batch = BatchSession.recover(m, conf, sessionStore, Some(mockApp))
+
+ batch.state shouldBe a[SessionState.Recovering]
+
+ batch.appIdKnown("appId")
+ verify(sessionStore, atLeastOnce()).save(
+ Matchers.eq(BatchSession.RECOVERY_SESSION_TYPE), anyObject())
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala b/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala
new file mode 100644
index 0000000..7fef3c2
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.batch
+
+import com.fasterxml.jackson.databind.{JsonMappingException, ObjectMapper}
+import org.scalatest.FunSpec
+
+import org.apache.livy.LivyBaseUnitTestSuite
+
+class CreateBatchRequestSpec extends FunSpec with LivyBaseUnitTestSuite {
+
+ private val mapper = new ObjectMapper()
+ .registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
+
+ describe("CreateBatchRequest") {
+
+ it("should have default values for fields after deserialization") {
+ val json = """{ "file" : "foo" }"""
+ val req = mapper.readValue(json, classOf[CreateBatchRequest])
+ assert(req.file === "foo")
+ assert(req.proxyUser === None)
+ assert(req.args === List())
+ assert(req.className === None)
+ assert(req.jars === List())
+ assert(req.pyFiles === List())
+ assert(req.files === List())
+ assert(req.driverMemory === None)
+ assert(req.driverCores === None)
+ assert(req.executorMemory === None)
+ assert(req.executorCores === None)
+ assert(req.numExecutors === None)
+ assert(req.archives === List())
+ assert(req.queue === None)
+ assert(req.name === None)
+ assert(req.conf === Map())
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/interactive/BaseInteractiveServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/interactive/BaseInteractiveServletSpec.scala b/server/src/test/scala/org/apache/livy/server/interactive/BaseInteractiveServletSpec.scala
new file mode 100644
index 0000000..b401a92
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/interactive/BaseInteractiveServletSpec.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.interactive
+
+import java.io.File
+import java.nio.file.Files
+
+import org.apache.commons.io.FileUtils
+import org.apache.spark.launcher.SparkLauncher
+
+import org.apache.livy.LivyConf
+import org.apache.livy.rsc.RSCConf
+import org.apache.livy.server.BaseSessionServletSpec
+import org.apache.livy.sessions.{Kind, SessionKindModule, Spark}
+
+abstract class BaseInteractiveServletSpec
+ extends BaseSessionServletSpec[InteractiveSession, InteractiveRecoveryMetadata] {
+
+ mapper.registerModule(new SessionKindModule())
+
+ protected var tempDir: File = _
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ if (tempDir != null) {
+ scala.util.Try(FileUtils.deleteDirectory(tempDir))
+ tempDir = null
+ }
+ }
+
+ override protected def createConf(): LivyConf = synchronized {
+ if (tempDir == null) {
+ tempDir = Files.createTempDirectory("client-test").toFile()
+ }
+ super.createConf()
+ .set(LivyConf.SESSION_STAGING_DIR, tempDir.toURI().toString())
+ .set(LivyConf.REPL_JARS, "dummy.jar")
+ .set(LivyConf.LIVY_SPARK_VERSION, "1.6.0")
+ .set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.10.5")
+ }
+
+ protected def createRequest(
+ inProcess: Boolean = true,
+ extraConf: Map[String, String] = Map(),
+ kind: Kind = Spark()): CreateInteractiveRequest = {
+ val classpath = sys.props("java.class.path")
+ val request = new CreateInteractiveRequest()
+ request.kind = kind
+ request.conf = extraConf ++ Map(
+ RSCConf.Entry.LIVY_JARS.key() -> "",
+ RSCConf.Entry.CLIENT_IN_PROCESS.key() -> inProcess.toString,
+ SparkLauncher.SPARK_MASTER -> "local",
+ SparkLauncher.DRIVER_EXTRA_CLASSPATH -> classpath,
+ SparkLauncher.EXECUTOR_EXTRA_CLASSPATH -> classpath
+ )
+ request
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala b/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala
new file mode 100644
index 0000000..a67c725
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.interactive
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import org.scalatest.FunSpec
+
+import org.apache.livy.LivyBaseUnitTestSuite
+import org.apache.livy.sessions.{PySpark, SessionKindModule}
+
+class CreateInteractiveRequestSpec extends FunSpec with LivyBaseUnitTestSuite {
+
+ private val mapper = new ObjectMapper()
+ .registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
+ .registerModule(new SessionKindModule())
+
+ describe("CreateInteractiveRequest") {
+
+ it("should have default values for fields after deserialization") {
+ val json = """{ "kind" : "pyspark" }"""
+ val req = mapper.readValue(json, classOf[CreateInteractiveRequest])
+ assert(req.kind === PySpark())
+ assert(req.proxyUser === None)
+ assert(req.jars === List())
+ assert(req.pyFiles === List())
+ assert(req.files === List())
+ assert(req.driverMemory === None)
+ assert(req.driverCores === None)
+ assert(req.executorMemory === None)
+ assert(req.executorCores === None)
+ assert(req.numExecutors === None)
+ assert(req.archives === List())
+ assert(req.queue === None)
+ assert(req.name === None)
+ assert(req.conf === Map())
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionServletSpec.scala b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionServletSpec.scala
new file mode 100644
index 0000000..372fe76
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionServletSpec.scala
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.interactive
+
+import java.util.concurrent.atomic.AtomicInteger
+import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
+
+import scala.collection.JavaConverters._
+import scala.concurrent.Future
+
+import org.json4s.jackson.Json4sScalaModule
+import org.mockito.Matchers._
+import org.mockito.Mockito._
+import org.mockito.invocation.InvocationOnMock
+import org.mockito.stubbing.Answer
+import org.scalatest.Entry
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.{ExecuteRequest, LivyConf}
+import org.apache.livy.client.common.HttpMessages.SessionInfo
+import org.apache.livy.rsc.driver.{Statement, StatementState}
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions._
+import org.apache.livy.utils.AppInfo
+
+class InteractiveSessionServletSpec extends BaseInteractiveServletSpec {
+
+ mapper.registerModule(new Json4sScalaModule())
+
+ class MockInteractiveSessionServlet(
+ sessionManager: InteractiveSessionManager,
+ conf: LivyConf)
+ extends InteractiveSessionServlet(sessionManager, mock[SessionStore], conf) {
+
+ private var statements = IndexedSeq[Statement]()
+
+ override protected def createSession(req: HttpServletRequest): InteractiveSession = {
+ val statementCounter = new AtomicInteger()
+
+ val session = mock[InteractiveSession]
+ when(session.kind).thenReturn(Spark())
+ when(session.appId).thenReturn(None)
+ when(session.appInfo).thenReturn(AppInfo())
+ when(session.logLines()).thenReturn(IndexedSeq())
+ when(session.state).thenReturn(SessionState.Idle())
+ when(session.stop()).thenReturn(Future.successful(()))
+ when(session.proxyUser).thenReturn(None)
+ when(session.statements).thenAnswer(
+ new Answer[IndexedSeq[Statement]]() {
+ override def answer(args: InvocationOnMock): IndexedSeq[Statement] = statements
+ })
+ when(session.executeStatement(any(classOf[ExecuteRequest]))).thenAnswer(
+ new Answer[Statement]() {
+ override def answer(args: InvocationOnMock): Statement = {
+ val id = statementCounter.getAndIncrement
+ val statement = new Statement(id, "1+1", StatementState.Available, "1")
+
+ statements :+= statement
+ statement
+ }
+ })
+ when(session.cancelStatement(anyInt())).thenAnswer(
+ new Answer[Unit] {
+ override def answer(args: InvocationOnMock): Unit = {
+ statements = IndexedSeq(
+ new Statement(statementCounter.get(), null, StatementState.Cancelled, null))
+ }
+ }
+ )
+
+ session
+ }
+
+ }
+
+ override def createServlet(): InteractiveSessionServlet = {
+ val conf = createConf()
+ val sessionManager = new InteractiveSessionManager(conf, mock[SessionStore], Some(Seq.empty))
+ new MockInteractiveSessionServlet(sessionManager, conf)
+ }
+
+ it("should setup and tear down an interactive session") {
+ jget[Map[String, Any]]("/") { data =>
+ data("sessions") should equal(Seq())
+ }
+
+ jpost[Map[String, Any]]("/", createRequest()) { data =>
+ header("Location") should equal("/0")
+ data("id") should equal (0)
+
+ val session = servlet.sessionManager.get(0)
+ session should be (defined)
+ }
+
+ jget[Map[String, Any]]("/0") { data =>
+ data("id") should equal (0)
+ data("state") should equal ("idle")
+
+ val batch = servlet.sessionManager.get(0)
+ batch should be (defined)
+ }
+
+ jpost[Map[String, Any]]("/0/statements", ExecuteRequest("foo")) { data =>
+ data("id") should be (0)
+ data("code") shouldBe "1+1"
+ data("progress") should be (0.0)
+ data("output") shouldBe 1
+ }
+
+ jget[Map[String, Any]]("/0/statements") { data =>
+ data("total_statements") should be (1)
+ data("statements").asInstanceOf[Seq[Map[String, Any]]](0)("id") should be (0)
+ }
+
+ jpost[Map[String, Any]]("/0/statements/0/cancel", null, HttpServletResponse.SC_OK) { data =>
+ data should equal(Map("msg" -> "canceled"))
+ }
+
+ jget[Map[String, Any]]("/0/statements") { data =>
+ data("total_statements") should be (1)
+ data("statements").asInstanceOf[Seq[Map[String, Any]]](0)("state") should be ("cancelled")
+ }
+
+ jdelete[Map[String, Any]]("/0") { data =>
+ data should equal (Map("msg" -> "deleted"))
+
+ val session = servlet.sessionManager.get(0)
+ session should not be defined
+ }
+ }
+
+ it("should show session properties") {
+ val id = 0
+ val appId = "appid"
+ val owner = "owner"
+ val proxyUser = "proxyUser"
+ val state = SessionState.Running()
+ val kind = Spark()
+ val appInfo = AppInfo(Some("DRIVER LOG URL"), Some("SPARK UI URL"))
+ val log = IndexedSeq[String]("log1", "log2")
+
+ val session = mock[InteractiveSession]
+ when(session.id).thenReturn(id)
+ when(session.appId).thenReturn(Some(appId))
+ when(session.owner).thenReturn(owner)
+ when(session.proxyUser).thenReturn(Some(proxyUser))
+ when(session.state).thenReturn(state)
+ when(session.kind).thenReturn(kind)
+ when(session.appInfo).thenReturn(appInfo)
+ when(session.logLines()).thenReturn(log)
+
+ val req = mock[HttpServletRequest]
+
+ val view = servlet.asInstanceOf[InteractiveSessionServlet].clientSessionView(session, req)
+ .asInstanceOf[SessionInfo]
+
+ view.id shouldEqual id
+ view.appId shouldEqual appId
+ view.owner shouldEqual owner
+ view.proxyUser shouldEqual proxyUser
+ view.state shouldEqual state.toString
+ view.kind shouldEqual kind.toString
+ view.appInfo should contain (Entry(AppInfo.DRIVER_LOG_URL_NAME, appInfo.driverLogUrl.get))
+ view.appInfo should contain (Entry(AppInfo.SPARK_UI_URL_NAME, appInfo.sparkUiUrl.get))
+ view.log shouldEqual log.asJava
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
new file mode 100644
index 0000000..d2ae9ae
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.interactive
+
+import java.net.URI
+
+import scala.concurrent.Await
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.apache.spark.launcher.SparkLauncher
+import org.json4s.{DefaultFormats, Extraction, JValue}
+import org.json4s.jackson.JsonMethods.parse
+import org.mockito.{Matchers => MockitoMatchers}
+import org.mockito.Matchers._
+import org.mockito.Mockito.{atLeastOnce, verify, when}
+import org.scalatest.{BeforeAndAfterAll, FunSpec, Matchers}
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.{ExecuteRequest, JobHandle, LivyBaseUnitTestSuite, LivyConf}
+import org.apache.livy.rsc.{PingJob, RSCClient, RSCConf}
+import org.apache.livy.rsc.driver.StatementState
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.{PySpark, SessionState, Spark}
+import org.apache.livy.utils.{AppInfo, SparkApp}
+
+class InteractiveSessionSpec extends FunSpec
+ with Matchers with BeforeAndAfterAll with LivyBaseUnitTestSuite {
+
+ private val livyConf = new LivyConf()
+ livyConf.set(LivyConf.REPL_JARS, "dummy.jar")
+ .set(LivyConf.LIVY_SPARK_VERSION, "1.6.0")
+ .set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.10.5")
+
+ implicit val formats = DefaultFormats
+
+ private var session: InteractiveSession = null
+
+ private def createSession(
+ sessionStore: SessionStore = mock[SessionStore],
+ mockApp: Option[SparkApp] = None): InteractiveSession = {
+ assume(sys.env.get("SPARK_HOME").isDefined, "SPARK_HOME is not set.")
+
+ val req = new CreateInteractiveRequest()
+ req.kind = PySpark()
+ req.driverMemory = Some("512m")
+ req.driverCores = Some(1)
+ req.executorMemory = Some("512m")
+ req.executorCores = Some(1)
+ req.name = Some("InteractiveSessionSpec")
+ req.conf = Map(
+ SparkLauncher.DRIVER_EXTRA_CLASSPATH -> sys.props("java.class.path"),
+ RSCConf.Entry.LIVY_JARS.key() -> ""
+ )
+ InteractiveSession.create(0, null, None, livyConf, req, sessionStore, mockApp)
+ }
+
+ private def executeStatement(code: String): JValue = {
+ val id = session.executeStatement(ExecuteRequest(code)).id
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ val s = session.getStatement(id).get
+ s.state.get() shouldBe StatementState.Available
+ parse(s.output)
+ }
+ }
+
+ override def afterAll(): Unit = {
+ if (session != null) {
+ Await.ready(session.stop(), 30 seconds)
+ session = null
+ }
+ super.afterAll()
+ }
+
+ private def withSession(desc: String)(fn: (InteractiveSession) => Unit): Unit = {
+ it(desc) {
+ assume(session != null, "No active session.")
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ session.state shouldBe a[SessionState.Idle]
+ }
+ fn(session)
+ }
+ }
+
+ describe("A spark session") {
+
+ it("should get scala version matched jars with livy.repl.jars") {
+ val testedJars = Seq(
+ "test_2.10-0.1.jar",
+ "local://dummy-path/test/test1_2.10-1.0.jar",
+ "file:///dummy-path/test/test2_2.11-1.0-SNAPSHOT.jar",
+ "hdfs:///dummy-path/test/test3.jar",
+ "non-jar",
+ "dummy.jar"
+ )
+ val livyConf = new LivyConf(false)
+ .set(LivyConf.REPL_JARS, testedJars.mkString(","))
+ .set(LivyConf.LIVY_SPARK_VERSION, "1.6.2")
+ .set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.10")
+ val properties = InteractiveSession.prepareBuilderProp(Map.empty, Spark(), livyConf)
+ assert(properties(LivyConf.SPARK_JARS).split(",").toSet === Set("test_2.10-0.1.jar",
+ "local://dummy-path/test/test1_2.10-1.0.jar",
+ "hdfs:///dummy-path/test/test3.jar",
+ "dummy.jar"))
+
+ livyConf.set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.11")
+ val properties1 = InteractiveSession.prepareBuilderProp(Map.empty, Spark(), livyConf)
+ assert(properties1(LivyConf.SPARK_JARS).split(",").toSet === Set(
+ "file:///dummy-path/test/test2_2.11-1.0-SNAPSHOT.jar",
+ "hdfs:///dummy-path/test/test3.jar",
+ "dummy.jar"))
+ }
+
+
+ it("should set rsc jars through livy conf") {
+ val rscJars = Set(
+ "dummy.jar",
+ "local:///dummy-path/dummy1.jar",
+ "file:///dummy-path/dummy2.jar",
+ "hdfs:///dummy-path/dummy3.jar")
+ val livyConf = new LivyConf(false)
+ .set(LivyConf.REPL_JARS, "dummy.jar")
+ .set(LivyConf.RSC_JARS, rscJars.mkString(","))
+ .set(LivyConf.LIVY_SPARK_VERSION, "1.6.2")
+ .set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.10")
+ val properties = InteractiveSession.prepareBuilderProp(Map.empty, Spark(), livyConf)
+ // if livy.rsc.jars is configured in LivyConf, it should be passed to RSCConf.
+ properties(RSCConf.Entry.LIVY_JARS.key()).split(",").toSet === rscJars
+
+ val rscJars1 = Set(
+ "foo.jar",
+ "local:///dummy-path/foo1.jar",
+ "file:///dummy-path/foo2.jar",
+ "hdfs:///dummy-path/foo3.jar")
+ val properties1 = InteractiveSession.prepareBuilderProp(
+ Map(RSCConf.Entry.LIVY_JARS.key() -> rscJars1.mkString(",")), Spark(), livyConf)
+ // if rsc jars are configured both in LivyConf and RSCConf, RSCConf should take precedence.
+ properties1(RSCConf.Entry.LIVY_JARS.key()).split(",").toSet === rscJars1
+ }
+
+ it("should start in the idle state") {
+ session = createSession()
+ session.state should (be(a[SessionState.Starting]) or be(a[SessionState.Idle]))
+ }
+
+ it("should update appId and appInfo and session store") {
+ val mockApp = mock[SparkApp]
+ val sessionStore = mock[SessionStore]
+ val session = createSession(sessionStore, Some(mockApp))
+
+ val expectedAppId = "APPID"
+ session.appIdKnown(expectedAppId)
+ session.appId shouldEqual Some(expectedAppId)
+
+ val expectedAppInfo = AppInfo(Some("DRIVER LOG URL"), Some("SPARK UI URL"))
+ session.infoChanged(expectedAppInfo)
+ session.appInfo shouldEqual expectedAppInfo
+
+ verify(sessionStore, atLeastOnce()).save(
+ MockitoMatchers.eq(InteractiveSession.RECOVERY_SESSION_TYPE), anyObject())
+ }
+
+ withSession("should execute `1 + 2` == 3") { session =>
+ val result = executeStatement("1 + 2")
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "3"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ withSession("should report an error if accessing an unknown variable") { session =>
+ val result = executeStatement("x")
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "error",
+ "execution_count" -> 1,
+ "ename" -> "NameError",
+ "evalue" -> "name 'x' is not defined",
+ "traceback" -> List(
+ "Traceback (most recent call last):\n",
+ "NameError: name 'x' is not defined\n"
+ )
+ ))
+
+ result should equal (expectedResult)
+ eventually(timeout(10 seconds), interval(30 millis)) {
+ session.state shouldBe a[SessionState.Idle]
+ }
+ }
+
+ withSession("should get statement progress along with statement result") { session =>
+ val code =
+ """
+ |from time import sleep
+ |sleep(3)
+ """.stripMargin
+ val statement = session.executeStatement(ExecuteRequest(code))
+ statement.progress should be (0.0)
+
+ eventually(timeout(10 seconds), interval(100 millis)) {
+ val s = session.getStatement(statement.id).get
+ s.state.get() shouldBe StatementState.Available
+ s.progress should be (1.0)
+ }
+ }
+
+ withSession("should error out the session if the interpreter dies") { session =>
+ session.executeStatement(ExecuteRequest("import os; os._exit(666)"))
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ session.state shouldBe a[SessionState.Error]
+ }
+ }
+ }
+
+ describe("recovery") {
+ it("should recover session") {
+ val conf = new LivyConf()
+ val sessionStore = mock[SessionStore]
+ val mockClient = mock[RSCClient]
+ when(mockClient.submit(any(classOf[PingJob]))).thenReturn(mock[JobHandle[Void]])
+ val m =
+ InteractiveRecoveryMetadata(
+ 78, None, "appTag", Spark(), 0, null, None, Some(URI.create("")))
+ val s = InteractiveSession.recover(m, conf, sessionStore, None, Some(mockClient))
+
+ s.state shouldBe a[SessionState.Recovering]
+
+ s.appIdKnown("appId")
+ verify(sessionStore, atLeastOnce()).save(
+ MockitoMatchers.eq(InteractiveSession.RECOVERY_SESSION_TYPE), anyObject())
+ }
+
+ it("should recover session to dead state if rscDriverUri is unknown") {
+ val conf = new LivyConf()
+ val sessionStore = mock[SessionStore]
+ val m = InteractiveRecoveryMetadata(
+ 78, Some("appId"), "appTag", Spark(), 0, null, None, None)
+ val s = InteractiveSession.recover(m, conf, sessionStore, None)
+
+ s.state shouldBe a[SessionState.Dead]
+ s.logLines().mkString should include("RSCDriver URI is unknown")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/interactive/JobApiSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/interactive/JobApiSpec.scala b/server/src/test/scala/org/apache/livy/server/interactive/JobApiSpec.scala
new file mode 100644
index 0000000..697a953
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/interactive/JobApiSpec.scala
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.interactive
+
+import java.io.File
+import java.net.URI
+import java.nio.ByteBuffer
+import java.nio.file.{Files, Paths}
+import javax.servlet.http.HttpServletResponse._
+
+import scala.concurrent.duration._
+import scala.io.Source
+import scala.language.postfixOps
+
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.{Job, JobHandle}
+import org.apache.livy.client.common.{BufferUtils, Serializer}
+import org.apache.livy.client.common.HttpMessages._
+import org.apache.livy.server.RemoteUserOverride
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.{InteractiveSessionManager, SessionState}
+import org.apache.livy.test.jobs.{Echo, GetCurrentUser}
+
+class JobApiSpec extends BaseInteractiveServletSpec {
+
+ private val PROXY = "__proxy__"
+
+ private var sessionId: Int = -1
+
+ override def createServlet(): InteractiveSessionServlet = {
+ val conf = createConf()
+ val sessionStore = mock[SessionStore]
+ val sessionManager = new InteractiveSessionManager(conf, sessionStore, Some(Seq.empty))
+ new InteractiveSessionServlet(sessionManager, sessionStore, conf) with RemoteUserOverride
+ }
+
+ def withSessionId(desc: String)(fn: (Int) => Unit): Unit = {
+ it(desc) {
+ assume(sessionId != -1, "No active session.")
+ fn(sessionId)
+ }
+ }
+
+ describe("Interactive Servlet") {
+
+ it("should create sessions") {
+ jpost[SessionInfo]("/", createRequest()) { data =>
+ waitForIdle(data.id)
+ header("Location") should equal("/0")
+ data.id should equal (0)
+ sessionId = data.id
+ }
+ }
+
+ withSessionId("should handle asynchronous jobs") { testJobSubmission(_, false) }
+
+ withSessionId("should handle synchronous jobs") { testJobSubmission(_, true) }
+
+ // Test that the file does get copied over to the live home dir on HDFS - does not test end
+ // to end that the RSCClient class copies it over to the app.
+ withSessionId("should support file uploads") { id =>
+ testResourceUpload("file", id)
+ }
+
+ withSessionId("should support jar uploads") { id =>
+ testResourceUpload("jar", id)
+ }
+
+ withSessionId("should monitor async Spark jobs") { sid =>
+ val ser = new Serializer()
+ val job = BufferUtils.toByteArray(ser.serialize(new Echo("hello")))
+ var jobId: Long = -1L
+ jpost[JobStatus](s"/$sid/submit-job", new SerializedJob(job)) { status =>
+ jobId = status.id
+ }
+
+ eventually(timeout(1 minute), interval(100 millis)) {
+ jget[JobStatus](s"/$sid/jobs/$jobId") { status =>
+ status.state should be (JobHandle.State.SUCCEEDED)
+ }
+ }
+ }
+
+ withSessionId("should update last activity on connect") { sid =>
+ val currentActivity = servlet.sessionManager.get(sid).get.lastActivity
+ jpost[SessionInfo](s"/$sid/connect", null, expectedStatus = SC_OK) { info =>
+ val newActivity = servlet.sessionManager.get(sid).get.lastActivity
+ assert(newActivity > currentActivity)
+ }
+ }
+
+ withSessionId("should tear down sessions") { id =>
+ jdelete[Map[String, Any]](s"/$id") { data =>
+ data should equal (Map("msg" -> "deleted"))
+ }
+ jget[Map[String, Any]]("/") { data =>
+ data("sessions") match {
+ case contents: Seq[_] => contents.size should equal (0)
+ case _ => fail("Response is not an array.")
+ }
+ }
+
+ // Make sure the session's staging directory was cleaned up.
+ assert(tempDir.listFiles().length === 0)
+ }
+
+ it("should support user impersonation") {
+ val headers = makeUserHeaders(PROXY)
+ jpost[SessionInfo]("/", createRequest(inProcess = false), headers = headers) { data =>
+ try {
+ waitForIdle(data.id)
+ data.owner should be (PROXY)
+ data.proxyUser should be (PROXY)
+ val user = runJob(data.id, new GetCurrentUser(), headers = headers)
+ user should be (PROXY)
+ } finally {
+ deleteSession(data.id)
+ }
+ }
+ }
+
+ it("should honor impersonation requests") {
+ val request = createRequest(inProcess = false)
+ request.proxyUser = Some(PROXY)
+ jpost[SessionInfo]("/", request, headers = adminHeaders) { data =>
+ try {
+ waitForIdle(data.id)
+ data.owner should be (ADMIN)
+ data.proxyUser should be (PROXY)
+ val user = runJob(data.id, new GetCurrentUser(), headers = adminHeaders)
+ user should be (PROXY)
+
+ // Test that files are uploaded to a new session directory.
+ assert(tempDir.listFiles().length === 0)
+ testResourceUpload("file", data.id)
+ } finally {
+ deleteSession(data.id)
+ assert(tempDir.listFiles().length === 0)
+ }
+ }
+ }
+
+ it("should respect config black list") {
+ jpost[SessionInfo]("/", createRequest(extraConf = BLACKLISTED_CONFIG),
+ expectedStatus = SC_BAD_REQUEST) { _ => }
+ }
+
+ }
+
+ private def waitForIdle(id: Int): Unit = {
+ eventually(timeout(1 minute), interval(100 millis)) {
+ jget[SessionInfo](s"/$id") { status =>
+ status.state should be (SessionState.Idle().toString())
+ }
+ }
+ }
+
+ private def deleteSession(id: Int): Unit = {
+ jdelete[Map[String, Any]](s"/$id", headers = adminHeaders) { _ => }
+ }
+
+ private def testResourceUpload(cmd: String, sessionId: Int): Unit = {
+ val f = File.createTempFile("uploadTestFile", cmd)
+ val conf = createConf()
+
+ Files.write(Paths.get(f.getAbsolutePath), "Test data".getBytes())
+
+ jupload[Unit](s"/$sessionId/upload-$cmd", Map(cmd -> f), expectedStatus = SC_OK) { _ =>
+ // There should be a single directory under the staging dir.
+ val subdirs = tempDir.listFiles()
+ assert(subdirs.length === 1)
+ val stagingDir = subdirs(0).toURI().toString()
+
+ val resultFile = new File(new URI(s"$stagingDir/${f.getName}"))
+ resultFile.deleteOnExit()
+ resultFile.exists() should be(true)
+ Source.fromFile(resultFile).mkString should be("Test data")
+ }
+ }
+
+ private def testJobSubmission(sid: Int, sync: Boolean): Unit = {
+ val result = runJob(sid, new Echo(42), sync = sync)
+ result should be (42)
+ }
+
+ private def runJob[T](
+ sid: Int,
+ job: Job[T],
+ sync: Boolean = false,
+ headers: Map[String, String] = defaultHeaders): T = {
+ val ser = new Serializer()
+ val jobData = BufferUtils.toByteArray(ser.serialize(job))
+ val route = if (sync) s"/$sid/submit-job" else s"/$sid/run-job"
+ var jobId: Long = -1L
+ jpost[JobStatus](route, new SerializedJob(jobData), headers = headers) { data =>
+ jobId = data.id
+ }
+
+ var result: Option[T] = None
+ eventually(timeout(1 minute), interval(100 millis)) {
+ jget[JobStatus](s"/$sid/jobs/$jobId") { status =>
+ status.id should be (jobId)
+ status.state should be (JobHandle.State.SUCCEEDED)
+ result = Some(ser.deserialize(ByteBuffer.wrap(status.result)).asInstanceOf[T])
+ }
+ }
+ result.getOrElse(throw new IllegalStateException())
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/interactive/SessionHeartbeatSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/interactive/SessionHeartbeatSpec.scala b/server/src/test/scala/org/apache/livy/server/interactive/SessionHeartbeatSpec.scala
new file mode 100644
index 0000000..12c8bbb
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/interactive/SessionHeartbeatSpec.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.interactive
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.mockito.Mockito.{never, verify, when}
+import org.scalatest.{FunSpec, Matchers}
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.LivyConf
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.{Session, SessionManager}
+import org.apache.livy.sessions.Session.RecoveryMetadata
+
+class SessionHeartbeatSpec extends FunSpec with Matchers {
+ describe("SessionHeartbeat") {
+ class TestHeartbeat(override val heartbeatTimeout: FiniteDuration) extends SessionHeartbeat {}
+
+ it("should not expire if heartbeat was never called.") {
+ val t = new TestHeartbeat(Duration.Zero)
+ t.heartbeatExpired shouldBe false
+ }
+
+ it("should expire if time has elapsed.") {
+ val t = new TestHeartbeat(Duration.fromNanos(1))
+ t.heartbeat()
+ eventually(timeout(2 nano), interval(1 nano)) {
+ t.heartbeatExpired shouldBe true
+ }
+ }
+
+ it("should not expire if time hasn't elapsed.") {
+ val t = new TestHeartbeat(Duration.create(1, DAYS))
+ t.heartbeat()
+ t.heartbeatExpired shouldBe false
+ }
+ }
+
+ describe("SessionHeartbeatWatchdog") {
+ abstract class TestSession extends Session(0, null, null) with SessionHeartbeat {}
+ class TestWatchdog(conf: LivyConf)
+ extends SessionManager[TestSession, RecoveryMetadata](
+ conf,
+ { _ => assert(false).asInstanceOf[TestSession] },
+ mock[SessionStore],
+ "test",
+ Some(Seq.empty))
+ with SessionHeartbeatWatchdog[TestSession, RecoveryMetadata] {}
+
+ it("should delete only expired sessions") {
+ val expiredSession: TestSession = mock[TestSession]
+ when(expiredSession.id).thenReturn(0)
+ when(expiredSession.heartbeatExpired).thenReturn(true)
+
+ val nonExpiredSession: TestSession = mock[TestSession]
+ when(nonExpiredSession.id).thenReturn(1)
+ when(nonExpiredSession.heartbeatExpired).thenReturn(false)
+
+ val n = new TestWatchdog(new LivyConf())
+
+ n.register(expiredSession)
+ n.register(nonExpiredSession)
+ n.deleteExpiredSessions()
+
+ verify(expiredSession).stop()
+ verify(nonExpiredSession, never).stop()
+ }
+ }
+}
[22/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/BaseProtocol.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/BaseProtocol.java b/rsc/src/main/java/org/apache/livy/rsc/BaseProtocol.java
new file mode 100644
index 0000000..c25e98f
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/BaseProtocol.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import org.apache.livy.Job;
+import org.apache.livy.rsc.rpc.RpcDispatcher;
+
+public abstract class BaseProtocol extends RpcDispatcher {
+
+ protected static class CancelJob {
+
+ public final String id;
+
+ CancelJob(String id) {
+ this.id = id;
+ }
+
+ CancelJob() {
+ this(null);
+ }
+
+ }
+
+ protected static class EndSession {
+
+ }
+
+ protected static class Error {
+
+ public final String cause;
+
+ public Error(Throwable cause) {
+ if (cause == null) {
+ this.cause = "";
+ } else {
+ this.cause = Utils.stackTraceAsString(cause);
+ }
+ }
+
+ public Error() {
+ this(null);
+ }
+
+ }
+
+ public static class BypassJobRequest {
+
+ public final String id;
+ public final byte[] serializedJob;
+ public final boolean synchronous;
+
+ public BypassJobRequest(String id, byte[] serializedJob, boolean synchronous) {
+ this.id = id;
+ this.serializedJob = serializedJob;
+ this.synchronous = synchronous;
+ }
+
+ public BypassJobRequest() {
+ this(null, null, false);
+ }
+
+ }
+
+ protected static class GetBypassJobStatus {
+
+ public final String id;
+
+ public GetBypassJobStatus(String id) {
+ this.id = id;
+ }
+
+ public GetBypassJobStatus() {
+ this(null);
+ }
+
+ }
+
+ protected static class JobRequest<T> {
+
+ public final String id;
+ public final Job<T> job;
+
+ public JobRequest(String id, Job<T> job) {
+ this.id = id;
+ this.job = job;
+ }
+
+ public JobRequest() {
+ this(null, null);
+ }
+
+ }
+
+ protected static class JobResult<T> {
+
+ public final String id;
+ public final T result;
+ public final String error;
+
+ public JobResult(String id, T result, Throwable error) {
+ this.id = id;
+ this.result = result;
+ this.error = error != null ? Utils.stackTraceAsString(error) : null;
+ }
+
+ public JobResult() {
+ this(null, null, null);
+ }
+
+ }
+
+ protected static class JobStarted {
+
+ public final String id;
+
+ public JobStarted(String id) {
+ this.id = id;
+ }
+
+ public JobStarted() {
+ this(null);
+ }
+
+ }
+
+ protected static class SyncJobRequest<T> {
+
+ public final Job<T> job;
+
+ public SyncJobRequest(Job<T> job) {
+ this.job = job;
+ }
+
+ public SyncJobRequest() {
+ this(null);
+ }
+
+ }
+
+ public static class RemoteDriverAddress {
+
+ public final String host;
+ public final int port;
+
+ public RemoteDriverAddress(String host, int port) {
+ this.host = host;
+ this.port = port;
+ }
+
+ public RemoteDriverAddress() {
+ this(null, -1);
+ }
+
+ }
+
+ public static class ReplJobRequest {
+
+ public final String code;
+
+ public ReplJobRequest(String code) {
+ this.code = code;
+ }
+
+ public ReplJobRequest() {
+ this(null);
+ }
+ }
+
+ public static class GetReplJobResults {
+ public boolean allResults;
+ public Integer from, size;
+
+ public GetReplJobResults(Integer from, Integer size) {
+ this.allResults = false;
+ this.from = from;
+ this.size = size;
+ }
+
+ public GetReplJobResults() {
+ this.allResults = true;
+ from = null;
+ size = null;
+ }
+ }
+
+ protected static class ReplState {
+
+ public final String state;
+
+ public ReplState(String state) {
+ this.state = state;
+ }
+
+ public ReplState() {
+ this(null);
+ }
+ }
+
+ public static class CancelReplJobRequest {
+ public final int id;
+
+ public CancelReplJobRequest(int id) {
+ this.id = id;
+ }
+
+ public CancelReplJobRequest() {
+ this(-1);
+ }
+ }
+
+ public static class InitializationError {
+
+ public final String stackTrace;
+
+ public InitializationError(String stackTrace) {
+ this.stackTrace = stackTrace;
+ }
+
+ public InitializationError() {
+ this(null);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/BypassJobStatus.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/BypassJobStatus.java b/rsc/src/main/java/org/apache/livy/rsc/BypassJobStatus.java
new file mode 100644
index 0000000..f62c14d
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/BypassJobStatus.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import org.apache.livy.JobHandle;
+
+public class BypassJobStatus {
+
+ public final JobHandle.State state;
+ public final byte[] result;
+ public final String error;
+
+ public BypassJobStatus(JobHandle.State state, byte[] result, String error) {
+ this.state = state;
+ this.result = result;
+ this.error = error;
+ }
+
+ BypassJobStatus() {
+ this(null, null, null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/ContextInfo.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/ContextInfo.java b/rsc/src/main/java/org/apache/livy/rsc/ContextInfo.java
new file mode 100644
index 0000000..96f69a4
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/ContextInfo.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+/**
+ * Information about a running RSC instance.
+ */
+class ContextInfo {
+
+ final String remoteAddress;
+ final int remotePort;
+ final String clientId;
+ final String secret;
+
+ ContextInfo(String remoteAddress, int remotePort, String clientId, String secret) {
+ this.remoteAddress = remoteAddress;
+ this.remotePort = remotePort;
+ this.clientId = clientId;
+ this.secret = secret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java b/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java
new file mode 100644
index 0000000..8f46c1e
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java
@@ -0,0 +1,457 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.attribute.PosixFilePermission.*;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.util.concurrent.Promise;
+import org.apache.spark.launcher.SparkLauncher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.client.common.TestUtils;
+import org.apache.livy.rsc.driver.RSCDriverBootstrapper;
+import org.apache.livy.rsc.rpc.Rpc;
+import org.apache.livy.rsc.rpc.RpcDispatcher;
+import org.apache.livy.rsc.rpc.RpcServer;
+
+import static org.apache.livy.rsc.RSCConf.Entry.*;
+
+/**
+ * Encapsulates code needed to launch a new Spark context and collect information about how
+ * to establish a client connection to it.
+ */
+class ContextLauncher {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ContextLauncher.class);
+ private static final AtomicInteger CHILD_IDS = new AtomicInteger();
+
+ private static final String SPARK_DEPLOY_MODE = "spark.submit.deployMode";
+ private static final String SPARK_JARS_KEY = "spark.jars";
+ private static final String SPARK_ARCHIVES_KEY = "spark.yarn.dist.archives";
+ private static final String SPARK_HOME_ENV = "SPARK_HOME";
+
+ static DriverProcessInfo create(RSCClientFactory factory, RSCConf conf)
+ throws IOException {
+ ContextLauncher launcher = new ContextLauncher(factory, conf);
+ return new DriverProcessInfo(launcher.promise, launcher.child.child);
+ }
+
+ private final Promise<ContextInfo> promise;
+ private final ScheduledFuture<?> timeout;
+ private final String clientId;
+ private final String secret;
+ private final ChildProcess child;
+ private final RSCConf conf;
+ private final RSCClientFactory factory;
+
+ private ContextLauncher(RSCClientFactory factory, RSCConf conf) throws IOException {
+ this.promise = factory.getServer().getEventLoopGroup().next().newPromise();
+ this.clientId = UUID.randomUUID().toString();
+ this.secret = factory.getServer().createSecret();
+ this.conf = conf;
+ this.factory = factory;
+
+ final RegistrationHandler handler = new RegistrationHandler();
+ try {
+ factory.getServer().registerClient(clientId, secret, handler);
+ String replMode = conf.get("repl");
+ boolean repl = replMode != null && replMode.equals("true");
+
+ conf.set(LAUNCHER_ADDRESS, factory.getServer().getAddress());
+ conf.set(LAUNCHER_PORT, factory.getServer().getPort());
+ conf.set(CLIENT_ID, clientId);
+ conf.set(CLIENT_SECRET, secret);
+
+ Utils.addListener(promise, new FutureListener<ContextInfo>() {
+ @Override
+ public void onFailure(Throwable error) throws Exception {
+ // If promise is cancelled or failed, make sure spark-submit is not leaked.
+ if (child != null) {
+ child.kill();
+ }
+ }
+ });
+
+ this.child = startDriver(conf, promise);
+
+ // Set up a timeout to fail the promise if we don't hear back from the context
+ // after a configurable timeout.
+ Runnable timeoutTask = new Runnable() {
+ @Override
+ public void run() {
+ connectTimeout(handler);
+ }
+ };
+ this.timeout = factory.getServer().getEventLoopGroup().schedule(timeoutTask,
+ conf.getTimeAsMs(RPC_CLIENT_HANDSHAKE_TIMEOUT), TimeUnit.MILLISECONDS);
+ } catch (Exception e) {
+ dispose(true);
+ throw Utils.propagate(e);
+ }
+ }
+
+ private void connectTimeout(RegistrationHandler handler) {
+ if (promise.tryFailure(new TimeoutException("Timed out waiting for context to start."))) {
+ handler.dispose();
+ }
+ dispose(true);
+ }
+
+ private void dispose(boolean forceKill) {
+ factory.getServer().unregisterClient(clientId);
+ try {
+ if (child != null) {
+ if (forceKill) {
+ child.kill();
+ } else {
+ child.detach();
+ }
+ }
+ } finally {
+ factory.unref();
+ }
+ }
+
+ private static ChildProcess startDriver(final RSCConf conf, Promise<?> promise)
+ throws IOException {
+ String livyJars = conf.get(LIVY_JARS);
+ if (livyJars == null) {
+ String livyHome = System.getenv("LIVY_HOME");
+ Utils.checkState(livyHome != null,
+ "Need one of LIVY_HOME or %s set.", LIVY_JARS.key());
+ File rscJars = new File(livyHome, "rsc-jars");
+ if (!rscJars.isDirectory()) {
+ rscJars = new File(livyHome, "rsc/target/jars");
+ }
+ Utils.checkState(rscJars.isDirectory(),
+ "Cannot find 'client-jars' directory under LIVY_HOME.");
+ List<String> jars = new ArrayList<>();
+ for (File f : rscJars.listFiles()) {
+ jars.add(f.getAbsolutePath());
+ }
+ livyJars = Utils.join(jars, ",");
+ }
+ merge(conf, SPARK_JARS_KEY, livyJars, ",");
+
+ String kind = conf.get(SESSION_KIND);
+ if ("sparkr".equals(kind)) {
+ merge(conf, SPARK_ARCHIVES_KEY, conf.get(RSCConf.Entry.SPARKR_PACKAGE), ",");
+ } else if ("pyspark".equals(kind)) {
+ merge(conf, "spark.submit.pyFiles", conf.get(RSCConf.Entry.PYSPARK_ARCHIVES), ",");
+ }
+
+ // Disable multiple attempts since the RPC server doesn't yet support multiple
+ // connections for the same registered app.
+ conf.set("spark.yarn.maxAppAttempts", "1");
+
+ // Let the launcher go away when launcher in yarn cluster mode. This avoids keeping lots
+ // of "small" Java processes lingering on the Livy server node.
+ conf.set("spark.yarn.submit.waitAppCompletion", "false");
+
+ if (!conf.getBoolean(CLIENT_IN_PROCESS) &&
+ // For tests which doesn't shutdown RscDriver gracefully, JaCoCo exec isn't dumped properly.
+ // Disable JaCoCo for this case.
+ !conf.getBoolean(TEST_STUCK_END_SESSION)) {
+ // For testing; propagate jacoco settings so that we also do coverage analysis
+ // on the launched driver. We replace the name of the main file ("main.exec")
+ // so that we don't end up fighting with the main test launcher.
+ String jacocoArgs = TestUtils.getJacocoArgs();
+ if (jacocoArgs != null) {
+ merge(conf, SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, jacocoArgs, " ");
+ }
+ }
+
+ final File confFile = writeConfToFile(conf);
+
+ if (ContextLauncher.mockSparkSubmit != null) {
+ LOG.warn("!!!! Using mock spark-submit. !!!!");
+ return new ChildProcess(conf, promise, ContextLauncher.mockSparkSubmit, confFile);
+ } else if (conf.getBoolean(CLIENT_IN_PROCESS)) {
+ // Mostly for testing things quickly. Do not do this in production.
+ LOG.warn("!!!! Running remote driver in-process. !!!!");
+ Runnable child = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ RSCDriverBootstrapper.main(new String[] { confFile.getAbsolutePath() });
+ } catch (Exception e) {
+ throw Utils.propagate(e);
+ }
+ }
+ };
+ return new ChildProcess(conf, promise, child, confFile);
+ } else {
+ final SparkLauncher launcher = new SparkLauncher();
+
+ // Spark 1.x does not support specifying deploy mode in conf and needs special handling.
+ String deployMode = conf.get(SPARK_DEPLOY_MODE);
+ if (deployMode != null) {
+ launcher.setDeployMode(deployMode);
+ }
+
+ launcher.setSparkHome(System.getenv(SPARK_HOME_ENV));
+ launcher.setAppResource("spark-internal");
+ launcher.setPropertiesFile(confFile.getAbsolutePath());
+ launcher.setMainClass(RSCDriverBootstrapper.class.getName());
+
+ if (conf.get(PROXY_USER) != null) {
+ launcher.addSparkArg("--proxy-user", conf.get(PROXY_USER));
+ }
+
+ return new ChildProcess(conf, promise, launcher.launch(), confFile);
+ }
+ }
+
+ private static void merge(RSCConf conf, String key, String livyConf, String sep) {
+ String confValue = Utils.join(Arrays.asList(livyConf, conf.get(key)), sep);
+ conf.set(key, confValue);
+ }
+
+ /**
+ * Write the configuration to a file readable only by the process's owner. Livy properties
+ * are written with an added prefix so that they can be loaded using SparkConf on the driver
+ * side.
+ *
+ * The default Spark configuration (from either SPARK_HOME or SPARK_CONF_DIR) is merged into
+ * the user configuration, so that defaults set by Livy's admin take effect when not overridden
+ * by the user.
+ */
+ private static File writeConfToFile(RSCConf conf) throws IOException {
+ Properties confView = new Properties();
+ for (Map.Entry<String, String> e : conf) {
+ String key = e.getKey();
+ if (!key.startsWith(RSCConf.SPARK_CONF_PREFIX)) {
+ key = RSCConf.LIVY_SPARK_PREFIX + key;
+ }
+ confView.setProperty(key, e.getValue());
+ }
+
+ // Load the default Spark configuration.
+ String confDir = System.getenv("SPARK_CONF_DIR");
+ if (confDir == null && System.getenv(SPARK_HOME_ENV) != null) {
+ confDir = System.getenv(SPARK_HOME_ENV) + File.separator + "conf";
+ }
+
+ if (confDir != null) {
+ File sparkDefaults = new File(confDir + File.separator + "spark-defaults.conf");
+ if (sparkDefaults.isFile()) {
+ Properties sparkConf = new Properties();
+ Reader r = new InputStreamReader(new FileInputStream(sparkDefaults), UTF_8);
+ try {
+ sparkConf.load(r);
+ } finally {
+ r.close();
+ }
+
+ for (String key : sparkConf.stringPropertyNames()) {
+ if (!confView.containsKey(key)) {
+ confView.put(key, sparkConf.getProperty(key));
+ }
+ }
+ }
+ }
+
+ File file = File.createTempFile("livyConf", ".properties");
+ Files.setPosixFilePermissions(file.toPath(), EnumSet.of(OWNER_READ, OWNER_WRITE));
+ //file.deleteOnExit();
+
+ Writer writer = new OutputStreamWriter(new FileOutputStream(file), UTF_8);
+ try {
+ confView.store(writer, "Livy App Context Configuration");
+ } finally {
+ writer.close();
+ }
+
+ return file;
+ }
+
+
+ private class RegistrationHandler extends BaseProtocol
+ implements RpcServer.ClientCallback {
+
+ volatile RemoteDriverAddress driverAddress;
+
+ private Rpc client;
+
+ @Override
+ public RpcDispatcher onNewClient(Rpc client) {
+ LOG.debug("New RPC client connected from {}.", client.getChannel());
+ this.client = client;
+ return this;
+ }
+
+ @Override
+ public void onSaslComplete(Rpc client) {
+ }
+
+ void dispose() {
+ if (client != null) {
+ client.close();
+ }
+ }
+
+ private void handle(ChannelHandlerContext ctx, RemoteDriverAddress msg) {
+ ContextInfo info = new ContextInfo(msg.host, msg.port, clientId, secret);
+ if (promise.trySuccess(info)) {
+ timeout.cancel(true);
+ LOG.debug("Received driver info for client {}: {}/{}.", client.getChannel(),
+ msg.host, msg.port);
+ } else {
+ LOG.warn("Connection established but promise is already finalized.");
+ }
+
+ ctx.executor().submit(new Runnable() {
+ @Override
+ public void run() {
+ dispose();
+ ContextLauncher.this.dispose(false);
+ }
+ });
+ }
+
+ }
+
+ private static class ChildProcess {
+
+ private final RSCConf conf;
+ private final Promise<?> promise;
+ private final Process child;
+ private final Thread monitor;
+ private final File confFile;
+
+ public ChildProcess(RSCConf conf, Promise<?> promise, Runnable child, File confFile) {
+ this.conf = conf;
+ this.promise = promise;
+ this.monitor = monitor(child, CHILD_IDS.incrementAndGet());
+ this.child = null;
+ this.confFile = confFile;
+ }
+
+ public ChildProcess(RSCConf conf, Promise<?> promise, final Process childProc, File confFile) {
+ int childId = CHILD_IDS.incrementAndGet();
+ this.conf = conf;
+ this.promise = promise;
+ this.child = childProc;
+ this.confFile = confFile;
+
+ Runnable monitorTask = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ int exitCode = child.waitFor();
+ if (exitCode != 0) {
+ LOG.warn("Child process exited with code {}.", exitCode);
+ fail(new IOException(String.format("Child process exited with code %d.", exitCode)));
+ }
+ } catch (InterruptedException ie) {
+ LOG.warn("Waiting thread interrupted, killing child process.");
+ Thread.interrupted();
+ child.destroy();
+ } catch (Exception e) {
+ LOG.warn("Exception while waiting for child process.", e);
+ }
+ }
+ };
+ this.monitor = monitor(monitorTask, childId);
+ }
+
+ private void fail(Throwable error) {
+ promise.tryFailure(error);
+ }
+
+ public void kill() {
+ if (child != null) {
+ child.destroy();
+ }
+ monitor.interrupt();
+ detach();
+
+ if (!monitor.isAlive()) {
+ return;
+ }
+
+ // Last ditch effort.
+ if (monitor.isAlive()) {
+ LOG.warn("Timed out shutting down remote driver, interrupting...");
+ monitor.interrupt();
+ }
+ }
+
+ public void detach() {
+ try {
+ monitor.join(conf.getTimeAsMs(CLIENT_SHUTDOWN_TIMEOUT));
+ } catch (InterruptedException ie) {
+ LOG.debug("Interrupted before driver thread was finished.");
+ }
+ }
+
+ private Thread monitor(final Runnable task, int childId) {
+ Runnable wrappedTask = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ task.run();
+ } finally {
+ confFile.delete();
+ }
+ }
+ };
+ Thread thread = new Thread(wrappedTask);
+ thread.setDaemon(true);
+ thread.setName("ContextLauncher-" + childId);
+ thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ LOG.warn("Child task threw exception.", e);
+ fail(e);
+ }
+ });
+ thread.start();
+ return thread;
+ }
+ }
+
+ // Just for testing.
+ static Process mockSparkSubmit;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/DriverProcessInfo.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/DriverProcessInfo.java b/rsc/src/main/java/org/apache/livy/rsc/DriverProcessInfo.java
new file mode 100644
index 0000000..ddc991c
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/DriverProcessInfo.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import io.netty.util.concurrent.Promise;
+
+/**
+ * Information about driver process and @{@link ContextInfo}
+ */
+public class DriverProcessInfo {
+
+ private Promise<ContextInfo> contextInfo;
+ private transient Process driverProcess;
+
+ public DriverProcessInfo(Promise<ContextInfo> contextInfo, Process driverProcess) {
+ this.contextInfo = contextInfo;
+ this.driverProcess = driverProcess;
+ }
+
+ public Promise<ContextInfo> getContextInfo() {
+ return contextInfo;
+ }
+
+ public Process getDriverProcess() {
+ return driverProcess;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/FutureListener.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/FutureListener.java b/rsc/src/main/java/org/apache/livy/rsc/FutureListener.java
new file mode 100644
index 0000000..9b99eae
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/FutureListener.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+/** A simplified future listener for netty futures. See Utils.addListener(). */
+public abstract class FutureListener<T> {
+
+ public void onSuccess(T result) throws Exception { }
+
+ public void onFailure(Throwable error) throws Exception { }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/JobHandleImpl.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/JobHandleImpl.java b/rsc/src/main/java/org/apache/livy/rsc/JobHandleImpl.java
new file mode 100644
index 0000000..0fc4ba2
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/JobHandleImpl.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import io.netty.util.concurrent.Promise;
+
+import org.apache.livy.JobHandle;
+import org.apache.livy.client.common.AbstractJobHandle;
+
+/**
+ * A handle to a submitted job. Allows for monitoring and controlling of the running remote job.
+ */
+class JobHandleImpl<T> extends AbstractJobHandle<T> {
+
+ private final RSCClient client;
+ private final String jobId;
+ private final Promise<T> promise;
+ private volatile State state;
+
+ JobHandleImpl(RSCClient client, Promise<T> promise, String jobId) {
+ this.client = client;
+ this.jobId = jobId;
+ this.promise = promise;
+ }
+
+ /** Requests a running job to be cancelled. */
+ @Override
+ public boolean cancel(boolean mayInterrupt) {
+ if (changeState(State.CANCELLED)) {
+ client.cancel(jobId);
+ promise.cancel(mayInterrupt);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public T get() throws ExecutionException, InterruptedException {
+ return promise.get();
+ }
+
+ @Override
+ public T get(long timeout, TimeUnit unit)
+ throws ExecutionException, InterruptedException, TimeoutException {
+ return promise.get(timeout, unit);
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return promise.isCancelled();
+ }
+
+ @Override
+ public boolean isDone() {
+ return promise.isDone();
+ }
+
+ @Override
+ protected T result() {
+ return promise.getNow();
+ }
+
+ @Override
+ protected Throwable error() {
+ return promise.cause();
+ }
+
+ @SuppressWarnings("unchecked")
+ void setSuccess(Object result) {
+ // The synchronization here is not necessary, but tests depend on it.
+ synchronized (listeners) {
+ promise.setSuccess((T) result);
+ changeState(State.SUCCEEDED);
+ }
+ }
+
+ void setFailure(Throwable error) {
+ // The synchronization here is not necessary, but tests depend on it.
+ synchronized (listeners) {
+ promise.setFailure(error);
+ changeState(State.FAILED);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/PingJob.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/PingJob.java b/rsc/src/main/java/org/apache/livy/rsc/PingJob.java
new file mode 100644
index 0000000..221f57f
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/PingJob.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+/** A job that can be used to check for liveness of the remote context. */
+public class PingJob implements Job<Void> {
+
+ @Override
+ public Void call(JobContext jc) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/RSCClient.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/RSCClient.java b/rsc/src/main/java/org/apache/livy/rsc/RSCClient.java
new file mode 100644
index 0000000..1b38467
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/RSCClient.java
@@ -0,0 +1,410 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.util.concurrent.GenericFutureListener;
+import io.netty.util.concurrent.ImmediateEventExecutor;
+import io.netty.util.concurrent.Promise;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobHandle;
+import org.apache.livy.LivyClient;
+import org.apache.livy.client.common.BufferUtils;
+import org.apache.livy.rsc.driver.AddFileJob;
+import org.apache.livy.rsc.driver.AddJarJob;
+import org.apache.livy.rsc.rpc.Rpc;
+
+import static org.apache.livy.rsc.RSCConf.Entry.*;
+
+public class RSCClient implements LivyClient {
+ private static final Logger LOG = LoggerFactory.getLogger(RSCClient.class);
+ private static final AtomicInteger EXECUTOR_GROUP_ID = new AtomicInteger();
+
+ private final RSCConf conf;
+ private final Promise<ContextInfo> contextInfoPromise;
+ private final Map<String, JobHandleImpl<?>> jobs;
+ private final ClientProtocol protocol;
+ private final Promise<Rpc> driverRpc;
+ private final int executorGroupId;
+ private final EventLoopGroup eventLoopGroup;
+ private final Promise<URI> serverUriPromise;
+
+ private ContextInfo contextInfo;
+ private Process driverProcess;
+ private volatile boolean isAlive;
+ private volatile String replState;
+
+ RSCClient(RSCConf conf, Promise<ContextInfo> ctx, Process driverProcess) throws IOException {
+ this.conf = conf;
+ this.contextInfoPromise = ctx;
+ this.driverProcess = driverProcess;
+ this.jobs = new ConcurrentHashMap<>();
+ this.protocol = new ClientProtocol();
+ this.driverRpc = ImmediateEventExecutor.INSTANCE.newPromise();
+ this.executorGroupId = EXECUTOR_GROUP_ID.incrementAndGet();
+ this.eventLoopGroup = new NioEventLoopGroup(
+ conf.getInt(RPC_MAX_THREADS),
+ Utils.newDaemonThreadFactory("RSCClient-" + executorGroupId + "-%d"));
+ this.serverUriPromise = ImmediateEventExecutor.INSTANCE.newPromise();
+
+ Utils.addListener(this.contextInfoPromise, new FutureListener<ContextInfo>() {
+ @Override
+ public void onSuccess(ContextInfo info) throws Exception {
+ connectToContext(info);
+ String url = String.format("rsc://%s:%s@%s:%d",
+ info.clientId, info.secret, info.remoteAddress, info.remotePort);
+ serverUriPromise.setSuccess(URI.create(url));
+ }
+
+ @Override
+ public void onFailure(Throwable error) {
+ connectionError(error);
+ serverUriPromise.setFailure(error);
+ }
+ });
+
+ isAlive = true;
+ }
+
+ public boolean isAlive() {
+ return isAlive;
+ }
+
+ public Process getDriverProcess() {
+ return driverProcess;
+ }
+
+ private synchronized void connectToContext(final ContextInfo info) throws Exception {
+ this.contextInfo = info;
+
+ try {
+ Promise<Rpc> promise = Rpc.createClient(conf,
+ eventLoopGroup,
+ info.remoteAddress,
+ info.remotePort,
+ info.clientId,
+ info.secret,
+ protocol);
+ Utils.addListener(promise, new FutureListener<Rpc>() {
+ @Override
+ public void onSuccess(Rpc rpc) throws Exception {
+ driverRpc.setSuccess(rpc);
+ Utils.addListener(rpc.getChannel().closeFuture(), new FutureListener<Void>() {
+ @Override
+ public void onSuccess(Void unused) {
+ if (isAlive) {
+ LOG.warn("Client RPC channel closed unexpectedly.");
+ try {
+ stop(false);
+ } catch (Exception e) { /* stop() itself prints warning. */ }
+ }
+ }
+ });
+ LOG.debug("Connected to context {} ({}, {}).", info.clientId,
+ rpc.getChannel(), executorGroupId);
+ }
+
+ @Override
+ public void onFailure(Throwable error) throws Exception {
+ driverRpc.setFailure(error);
+ connectionError(error);
+ }
+ });
+ } catch (Exception e) {
+ connectionError(e);
+ }
+ }
+
+ private void connectionError(Throwable error) {
+ LOG.error("Failed to connect to context.", error);
+ try {
+ stop(false);
+ } catch (Exception e) { /* stop() itself prints warning. */ }
+ }
+
+ private <T> io.netty.util.concurrent.Future<T> deferredCall(final Object msg,
+ final Class<T> retType) {
+ if (driverRpc.isSuccess()) {
+ try {
+ return driverRpc.get().call(msg, retType);
+ } catch (Exception ie) {
+ throw Utils.propagate(ie);
+ }
+ }
+
+ // No driver RPC yet, so install a listener and return a promise that will be ready when
+ // the driver is up and the message is actually delivered.
+ final Promise<T> promise = eventLoopGroup.next().newPromise();
+ final FutureListener<T> callListener = new FutureListener<T>() {
+ @Override
+ public void onSuccess(T value) throws Exception {
+ promise.setSuccess(value);
+ }
+
+ @Override
+ public void onFailure(Throwable error) throws Exception {
+ promise.setFailure(error);
+ }
+ };
+
+ Utils.addListener(driverRpc, new FutureListener<Rpc>() {
+ @Override
+ public void onSuccess(Rpc rpc) throws Exception {
+ Utils.addListener(rpc.call(msg, retType), callListener);
+ }
+
+ @Override
+ public void onFailure(Throwable error) throws Exception {
+ promise.setFailure(error);
+ }
+ });
+ return promise;
+ }
+
+ public Future<URI> getServerUri() {
+ return serverUriPromise;
+ }
+
+ @Override
+ public <T> JobHandle<T> submit(Job<T> job) {
+ return protocol.submit(job);
+ }
+
+ @Override
+ public <T> Future<T> run(Job<T> job) {
+ return protocol.run(job);
+ }
+
+ @Override
+ public synchronized void stop(boolean shutdownContext) {
+ if (isAlive) {
+ isAlive = false;
+ try {
+ this.contextInfoPromise.cancel(true);
+
+ if (shutdownContext && driverRpc.isSuccess()) {
+ protocol.endSession();
+
+ // Because the remote context won't really reply to the end session message -
+ // since it closes the channel while handling it, we wait for the RPC's channel
+ // to close instead.
+ long stopTimeout = conf.getTimeAsMs(CLIENT_SHUTDOWN_TIMEOUT);
+ driverRpc.get().getChannel().closeFuture().get(stopTimeout,
+ TimeUnit.MILLISECONDS);
+ }
+ } catch (Exception e) {
+ LOG.warn("Exception while waiting for end session reply.", e);
+ Utils.propagate(e);
+ } finally {
+ if (driverRpc.isSuccess()) {
+ try {
+ driverRpc.get().close();
+ } catch (Exception e) {
+ LOG.warn("Error stopping RPC.", e);
+ }
+ }
+
+ // Report failure for all pending jobs, so that clients can react.
+ for (Map.Entry<String, JobHandleImpl<?>> e : jobs.entrySet()) {
+ LOG.info("Failing pending job {} due to shutdown.", e.getKey());
+ e.getValue().setFailure(new IOException("RSCClient instance stopped."));
+ }
+
+ eventLoopGroup.shutdownGracefully();
+ }
+ if (contextInfo != null) {
+ LOG.debug("Disconnected from context {}, shutdown = {}.", contextInfo.clientId,
+ shutdownContext);
+ }
+ }
+ }
+
+ @Override
+ public Future<?> uploadJar(File jar) {
+ throw new UnsupportedOperationException("Use addJar to add the jar to the remote context!");
+ }
+
+ @Override
+ public Future<?> addJar(URI uri) {
+ return submit(new AddJarJob(uri.toString()));
+ }
+
+ @Override
+ public Future<?> uploadFile(File file) {
+ throw new UnsupportedOperationException("Use addFile to add the file to the remote context!");
+ }
+
+ @Override
+ public Future<?> addFile(URI uri) {
+ return submit(new AddFileJob(uri.toString()));
+ }
+
+ public String bypass(ByteBuffer serializedJob, boolean sync) {
+ return protocol.bypass(serializedJob, sync);
+ }
+
+ public Future<BypassJobStatus> getBypassJobStatus(String id) {
+ return protocol.getBypassJobStatus(id);
+ }
+
+ public void cancel(String jobId) {
+ protocol.cancel(jobId);
+ }
+
+ ContextInfo getContextInfo() {
+ return contextInfo;
+ }
+
+ public Future<Integer> submitReplCode(String code) throws Exception {
+ return deferredCall(new BaseProtocol.ReplJobRequest(code), Integer.class);
+ }
+
+ public void cancelReplCode(int statementId) throws Exception {
+ deferredCall(new BaseProtocol.CancelReplJobRequest(statementId), Void.class);
+ }
+
+ public Future<ReplJobResults> getReplJobResults(Integer from, Integer size) throws Exception {
+ return deferredCall(new BaseProtocol.GetReplJobResults(from, size), ReplJobResults.class);
+ }
+
+ public Future<ReplJobResults> getReplJobResults() throws Exception {
+ return deferredCall(new BaseProtocol.GetReplJobResults(), ReplJobResults.class);
+ }
+
+ /**
+ * @return Return the repl state. If this's not connected to a repl session, it will return null.
+ */
+ public String getReplState() {
+ return replState;
+ }
+
+ private class ClientProtocol extends BaseProtocol {
+
+ <T> JobHandleImpl<T> submit(Job<T> job) {
+ final String jobId = UUID.randomUUID().toString();
+ Object msg = new JobRequest<T>(jobId, job);
+
+ final Promise<T> promise = eventLoopGroup.next().newPromise();
+ final JobHandleImpl<T> handle = new JobHandleImpl<T>(RSCClient.this,
+ promise, jobId);
+ jobs.put(jobId, handle);
+
+ final io.netty.util.concurrent.Future<Void> rpc = deferredCall(msg, Void.class);
+ LOG.debug("Sending JobRequest[{}].", jobId);
+
+ Utils.addListener(rpc, new FutureListener<Void>() {
+ @Override
+ public void onSuccess(Void unused) throws Exception {
+ handle.changeState(JobHandle.State.QUEUED);
+ }
+
+ @Override
+ public void onFailure(Throwable error) throws Exception {
+ error.printStackTrace();
+ promise.tryFailure(error);
+ }
+ });
+ promise.addListener(new GenericFutureListener<Promise<T>>() {
+ @Override
+ public void operationComplete(Promise<T> p) {
+ if (jobId != null) {
+ jobs.remove(jobId);
+ }
+ if (p.isCancelled() && !rpc.isDone()) {
+ rpc.cancel(true);
+ }
+ }
+ });
+ return handle;
+ }
+
+ @SuppressWarnings("unchecked")
+ <T> Future<T> run(Job<T> job) {
+ return (Future<T>) deferredCall(new SyncJobRequest(job), Object.class);
+ }
+
+ String bypass(ByteBuffer serializedJob, boolean sync) {
+ String jobId = UUID.randomUUID().toString();
+ Object msg = new BypassJobRequest(jobId, BufferUtils.toByteArray(serializedJob), sync);
+ deferredCall(msg, Void.class);
+ return jobId;
+ }
+
+ Future<BypassJobStatus> getBypassJobStatus(String id) {
+ return deferredCall(new GetBypassJobStatus(id), BypassJobStatus.class);
+ }
+
+ void cancel(String jobId) {
+ deferredCall(new CancelJob(jobId), Void.class);
+ }
+
+ Future<?> endSession() {
+ return deferredCall(new EndSession(), Void.class);
+ }
+
+ private void handle(ChannelHandlerContext ctx, InitializationError msg) {
+ LOG.warn("Error reported from remote driver: %s", msg.stackTrace);
+ }
+
+ private void handle(ChannelHandlerContext ctx, JobResult msg) {
+ JobHandleImpl<?> handle = jobs.remove(msg.id);
+ if (handle != null) {
+ LOG.info("Received result for {}", msg.id);
+ // TODO: need a better exception for this.
+ Throwable error = msg.error != null ? new RuntimeException(msg.error) : null;
+ if (error == null) {
+ handle.setSuccess(msg.result);
+ } else {
+ handle.setFailure(error);
+ }
+ } else {
+ LOG.warn("Received result for unknown job {}", msg.id);
+ }
+ }
+
+ private void handle(ChannelHandlerContext ctx, JobStarted msg) {
+ JobHandleImpl<?> handle = jobs.get(msg.id);
+ if (handle != null) {
+ handle.changeState(JobHandle.State.STARTED);
+ } else {
+ LOG.warn("Received event for unknown job {}", msg.id);
+ }
+ }
+
+ private void handle(ChannelHandlerContext ctx, ReplState msg) {
+ LOG.trace("Received repl state for {}", msg.state);
+ replState = msg.state;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/RSCClientFactory.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/RSCClientFactory.java b/rsc/src/main/java/org/apache/livy/rsc/RSCClientFactory.java
new file mode 100644
index 0000000..c6327e2
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/RSCClientFactory.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import io.netty.util.concurrent.ImmediateEventExecutor;
+import io.netty.util.concurrent.Promise;
+
+import org.apache.livy.LivyClient;
+import org.apache.livy.LivyClientFactory;
+import org.apache.livy.rsc.rpc.RpcServer;
+
+/**
+ * Factory for RSC clients.
+ */
+public final class RSCClientFactory implements LivyClientFactory {
+
+ private final AtomicInteger refCount = new AtomicInteger();
+ private RpcServer server = null;
+
+ /**
+ * Creates a local Livy client if the URI has the "rsc" scheme.
+ * <p>
+ * If the URI contains user information, host and port, the library will try to connect to an
+ * existing RSC instance with the provided information, and most of the provided configuration
+ * will be ignored.
+ * <p>
+ * Otherwise, a new Spark context will be started with the given configuration.
+ */
+ @Override
+ public LivyClient createClient(URI uri, Properties config) {
+ if (!"rsc".equals(uri.getScheme())) {
+ return null;
+ }
+
+ RSCConf lconf = new RSCConf(config);
+
+ boolean needsServer = false;
+ try {
+ Promise<ContextInfo> info;
+ Process driverProcess = null;
+ if (uri.getUserInfo() != null && uri.getHost() != null && uri.getPort() > 0) {
+ info = createContextInfo(uri);
+ } else {
+ needsServer = true;
+ ref(lconf);
+ DriverProcessInfo processInfo = ContextLauncher.create(this, lconf);
+ info = processInfo.getContextInfo();
+ driverProcess = processInfo.getDriverProcess();
+ }
+ return new RSCClient(lconf, info, driverProcess);
+ } catch (Exception e) {
+ if (needsServer) {
+ unref();
+ }
+ throw Utils.propagate(e);
+ }
+ }
+
+ RpcServer getServer() {
+ return server;
+ }
+
+ private synchronized void ref(RSCConf config) throws IOException {
+ if (refCount.get() != 0) {
+ refCount.incrementAndGet();
+ return;
+ }
+
+ Utils.checkState(server == null, "Server already running but ref count is 0.");
+ if (server == null) {
+ try {
+ server = new RpcServer(config);
+ } catch (InterruptedException ie) {
+ throw Utils.propagate(ie);
+ }
+ }
+
+ refCount.incrementAndGet();
+ }
+
+ synchronized void unref() {
+ if (refCount.decrementAndGet() == 0) {
+ server.close();
+ server = null;
+ }
+ }
+
+ private static Promise<ContextInfo> createContextInfo(final URI uri) {
+ String[] userInfo = uri.getUserInfo().split(":", 2);
+ ImmediateEventExecutor executor = ImmediateEventExecutor.INSTANCE;
+ Promise<ContextInfo> promise = executor.newPromise();
+ promise.setSuccess(new ContextInfo(uri.getHost(), uri.getPort(), userInfo[0], userInfo[1]));
+ return promise;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/RSCConf.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/RSCConf.java b/rsc/src/main/java/org/apache/livy/rsc/RSCConf.java
new file mode 100644
index 0000000..c560aed
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/RSCConf.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import java.io.IOException;
+import java.net.Inet4Address;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import javax.security.sasl.Sasl;
+
+import org.apache.livy.client.common.ClientConf;
+
+public class RSCConf extends ClientConf<RSCConf> {
+
+ public static final String SPARK_CONF_PREFIX = "spark.";
+ public static final String LIVY_SPARK_PREFIX = SPARK_CONF_PREFIX + "__livy__.";
+
+ private static final String RSC_CONF_PREFIX = "livy.rsc.";
+
+ public static enum Entry implements ConfEntry {
+ CLIENT_ID("client.auth.id", null),
+ CLIENT_SECRET("client.auth.secret", null),
+ CLIENT_IN_PROCESS("client.do-not-use.run-driver-in-process", false),
+ CLIENT_SHUTDOWN_TIMEOUT("client.shutdown-timeout", "10s"),
+ DRIVER_CLASS("driver-class", null),
+ SESSION_KIND("session.kind", null),
+
+ LIVY_JARS("jars", null),
+ SPARKR_PACKAGE("sparkr.package", null),
+ PYSPARK_ARCHIVES("pyspark.archives", null),
+
+ // Address for the RSC driver to connect back with it's connection info.
+ LAUNCHER_ADDRESS("launcher.address", null),
+ LAUNCHER_PORT_RANGE("launcher.port.range", "10000~10010"),
+ // Setting up of this propety by user has no benefit. It is currently being used
+ // to pass port information from ContextLauncher to RSCDriver
+ LAUNCHER_PORT("launcher.port", -1),
+ // How long will the RSC wait for a connection for a Livy server before shutting itself down.
+ SERVER_IDLE_TIMEOUT("server.idle-timeout", "10m"),
+
+ PROXY_USER("proxy-user", null),
+
+ RPC_SERVER_ADDRESS("rpc.server.address", null),
+ RPC_CLIENT_HANDSHAKE_TIMEOUT("server.connect.timeout", "90s"),
+ RPC_CLIENT_CONNECT_TIMEOUT("client.connect.timeout", "10s"),
+ RPC_CHANNEL_LOG_LEVEL("channel.log.level", null),
+ RPC_MAX_MESSAGE_SIZE("rpc.max.size", 50 * 1024 * 1024),
+ RPC_MAX_THREADS("rpc.threads", 8),
+ RPC_SECRET_RANDOM_BITS("secret.bits", 256),
+
+ SASL_MECHANISMS("rpc.sasl.mechanisms", "DIGEST-MD5"),
+ SASL_QOP("rpc.sasl.qop", null),
+
+ TEST_STUCK_END_SESSION("test.do-not-use.stuck-end-session", false),
+ TEST_STUCK_START_DRIVER("test.do-not-use.stuck-start-driver", false),
+
+ JOB_CANCEL_TRIGGER_INTERVAL("job-cancel.trigger-interval", "100ms"),
+ JOB_CANCEL_TIMEOUT("job-cancel.timeout", "30s"),
+
+ RETAINED_STATEMENT_NUMBER("retained-statements", 100);
+
+ private final String key;
+ private final Object dflt;
+
+ private Entry(String key, Object dflt) {
+ this.key = RSC_CONF_PREFIX + key;
+ this.dflt = dflt;
+ }
+
+ @Override
+ public String key() { return key; }
+
+ @Override
+ public Object dflt() { return dflt; }
+ }
+
+ public RSCConf() {
+ this(new Properties());
+ }
+
+ public RSCConf(Properties config) {
+ super(config);
+ }
+
+ public Map<String, String> getSaslOptions() {
+ Map<String, String> opts = new HashMap<>();
+
+ // TODO: add more options?
+ String qop = get(Entry.SASL_QOP);
+ if (qop != null) {
+ opts.put(Sasl.QOP, qop);
+ }
+
+ return opts;
+ }
+
+ public String findLocalAddress() throws IOException {
+ InetAddress address = InetAddress.getLocalHost();
+ if (address.isLoopbackAddress()) {
+ // Address resolves to something like 127.0.1.1, which happens on Debian;
+ // try to find a better address using the local network interfaces
+ Enumeration<NetworkInterface> ifaces = NetworkInterface.getNetworkInterfaces();
+ while (ifaces.hasMoreElements()) {
+ NetworkInterface ni = ifaces.nextElement();
+ Enumeration<InetAddress> addrs = ni.getInetAddresses();
+ while (addrs.hasMoreElements()) {
+ InetAddress addr = addrs.nextElement();
+ if (!addr.isLinkLocalAddress() && !addr.isLoopbackAddress()
+ && addr instanceof Inet4Address) {
+ // We've found an address that looks reasonable!
+ LOG.warn("Your hostname, {}, resolves to a loopback address; using {} "
+ + " instead (on interface {})", address.getHostName(), addr.getHostAddress(),
+ ni.getName());
+ LOG.warn("Set '{}' if you need to bind to another address.",
+ Entry.RPC_SERVER_ADDRESS.key);
+ return addr.getHostAddress();
+ }
+ }
+ }
+ }
+
+ LOG.warn("Your hostname, {}, resolves to a loopback address, but we couldn't find "
+ + "any external IP address!", address.getCanonicalHostName());
+ LOG.warn("Set {} if you need to bind to another address.",
+ Entry.RPC_SERVER_ADDRESS.key);
+ return address.getCanonicalHostName();
+ }
+
+ private static final Map<String, DeprecatedConf> configsWithAlternatives
+ = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{
+ put(RSCConf.Entry.CLIENT_IN_PROCESS.key, DepConf.CLIENT_IN_PROCESS);
+ put(RSCConf.Entry.CLIENT_SHUTDOWN_TIMEOUT.key, DepConf.CLIENT_SHUTDOWN_TIMEOUT);
+ put(RSCConf.Entry.DRIVER_CLASS.key, DepConf.DRIVER_CLASS);
+ put(RSCConf.Entry.SERVER_IDLE_TIMEOUT.key, DepConf.SERVER_IDLE_TIMEOUT);
+ put(RSCConf.Entry.PROXY_USER.key, DepConf.PROXY_USER);
+ put(RSCConf.Entry.TEST_STUCK_END_SESSION.key, DepConf.TEST_STUCK_END_SESSION);
+ put(RSCConf.Entry.TEST_STUCK_START_DRIVER.key, DepConf.TEST_STUCK_START_DRIVER);
+ put(RSCConf.Entry.JOB_CANCEL_TRIGGER_INTERVAL.key, DepConf.JOB_CANCEL_TRIGGER_INTERVAL);
+ put(RSCConf.Entry.JOB_CANCEL_TIMEOUT.key, DepConf.JOB_CANCEL_TIMEOUT);
+ put(RSCConf.Entry.RETAINED_STATEMENT_NUMBER.key, DepConf.RETAINED_STATEMENT_NUMBER);
+ }});
+
+ // Maps deprecated key to DeprecatedConf with the same key.
+ // There are no deprecated configs without alternatives currently.
+ private static final Map<String, DeprecatedConf> deprecatedConfigs
+ = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>());
+
+ protected Map<String, DeprecatedConf> getConfigsWithAlternatives() {
+ return configsWithAlternatives;
+ }
+
+ protected Map<String, DeprecatedConf> getDeprecatedConfigs() {
+ return deprecatedConfigs;
+ }
+
+ static enum DepConf implements DeprecatedConf {
+ CLIENT_IN_PROCESS("client.do_not_use.run_driver_in_process", "0.4"),
+ CLIENT_SHUTDOWN_TIMEOUT("client.shutdown_timeout", "0.4"),
+ DRIVER_CLASS("driver_class", "0.4"),
+ SERVER_IDLE_TIMEOUT("server.idle_timeout", "0.4"),
+ PROXY_USER("proxy_user", "0.4"),
+ TEST_STUCK_END_SESSION("test.do_not_use.stuck_end_session", "0.4"),
+ TEST_STUCK_START_DRIVER("test.do_not_use.stuck_start_driver", "0.4"),
+ JOB_CANCEL_TRIGGER_INTERVAL("job_cancel.trigger_interval", "0.4"),
+ JOB_CANCEL_TIMEOUT("job_cancel.timeout", "0.4"),
+ RETAINED_STATEMENT_NUMBER("retained_statements", "0.4");
+
+ private final String key;
+ private final String version;
+ private final String deprecationMessage;
+
+ private DepConf(String key, String version) {
+ this(key, version, "");
+ }
+
+ private DepConf(String key, String version, String deprecationMessage) {
+ this.key = RSC_CONF_PREFIX + key;
+ this.version = version;
+ this.deprecationMessage = deprecationMessage;
+ }
+
+ @Override
+ public String key() { return key; }
+
+ @Override
+ public String version() { return version; }
+
+ @Override
+ public String deprecationMessage() { return deprecationMessage; }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/ReplJobResults.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/ReplJobResults.java b/rsc/src/main/java/org/apache/livy/rsc/ReplJobResults.java
new file mode 100644
index 0000000..6717c59
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/ReplJobResults.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.rsc;
+
+import org.apache.livy.rsc.driver.Statement;
+
+public class ReplJobResults {
+ public final Statement[] statements;
+
+ public ReplJobResults(Statement[] statements) {
+ this.statements = statements;
+ }
+
+ public ReplJobResults() {
+ this(null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/Utils.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/Utils.java b/rsc/src/main/java/org/apache/livy/rsc/Utils.java
new file mode 100644
index 0000000..d2c0059
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/Utils.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+
+/**
+ * A few simple utility functions used by the code, mostly to avoid a direct dependency
+ * on Guava.
+ */
+public class Utils {
+
+ public static void checkArgument(boolean condition) {
+ if (!condition) {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ public static void checkArgument(boolean condition, String msg, Object... args) {
+ if (!condition) {
+ throw new IllegalArgumentException(String.format(msg, args));
+ }
+ }
+
+ public static void checkState(boolean condition, String msg, Object... args) {
+ if (!condition) {
+ throw new IllegalStateException(String.format(msg, args));
+ }
+ }
+
+ public static void checkNotNull(Object o) {
+ if (o == null) {
+ throw new NullPointerException();
+ }
+ }
+
+ public static RuntimeException propagate(Throwable t) {
+ if (t instanceof RuntimeException) {
+ throw (RuntimeException) t;
+ } else {
+ throw new RuntimeException(t);
+ }
+ }
+
+ public static ThreadFactory newDaemonThreadFactory(final String nameFormat) {
+ return new ThreadFactory() {
+
+ private final AtomicInteger threadId = new AtomicInteger();
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setName(String.format(nameFormat, threadId.incrementAndGet()));
+ t.setDaemon(true);
+ return t;
+ }
+
+ };
+ }
+
+ public static String join(Iterable<String> strs, String sep) {
+ StringBuilder sb = new StringBuilder();
+ for (String s : strs) {
+ if (s != null && !s.isEmpty()) {
+ sb.append(s).append(sep);
+ }
+ }
+ if (sb.length() > 0) {
+ sb.setLength(sb.length() - sep.length());
+ }
+ return sb.toString();
+ }
+
+ public static String stackTraceAsString(Throwable t) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(t.getClass().getName()).append(": ").append(t.getMessage());
+ for (StackTraceElement e : t.getStackTrace()) {
+ sb.append("\n");
+ sb.append(e.toString());
+ }
+ return sb.toString();
+ }
+
+ public static <T> void addListener(Future<T> future, final FutureListener<T> lsnr) {
+ future.addListener(new GenericFutureListener<Future<T>>() {
+ @Override
+ public void operationComplete(Future<T> f) throws Exception {
+ if (f.isSuccess()) {
+ lsnr.onSuccess(f.get());
+ } else {
+ lsnr.onFailure(f.cause());
+ }
+ }
+ });
+ }
+
+ private Utils() { }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/AddFileJob.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/AddFileJob.java b/rsc/src/main/java/org/apache/livy/rsc/driver/AddFileJob.java
new file mode 100644
index 0000000..cc75b6c
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/AddFileJob.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.rsc.driver;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class AddFileJob implements Job<Object> {
+
+ private final String path;
+
+ AddFileJob() {
+ this(null);
+}
+
+ public AddFileJob(String path) {
+ this.path = path;
+}
+
+ @Override
+ public Object call(JobContext jc) throws Exception {
+ JobContextImpl jobContextImpl = (JobContextImpl)jc;
+ jobContextImpl.addFile(path);
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/AddJarJob.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/AddJarJob.java b/rsc/src/main/java/org/apache/livy/rsc/driver/AddJarJob.java
new file mode 100644
index 0000000..c455e6e
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/AddJarJob.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.driver;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class AddJarJob implements Job<Object> {
+
+ private final String path;
+
+ // For serialization.
+ private AddJarJob() {
+ this(null);
+ }
+
+ public AddJarJob(String path) {
+ this.path = path;
+ }
+
+ @Override
+ public Object call(JobContext jc) throws Exception {
+ JobContextImpl jobContextImpl = (JobContextImpl)jc;
+ jobContextImpl.addJarOrPyFile(path);
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/BypassJob.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/BypassJob.java b/rsc/src/main/java/org/apache/livy/rsc/driver/BypassJob.java
new file mode 100644
index 0000000..f0d14c6
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/BypassJob.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.driver;
+
+import java.nio.ByteBuffer;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+import org.apache.livy.client.common.BufferUtils;
+import org.apache.livy.client.common.Serializer;
+
+class BypassJob implements Job<byte[]> {
+
+ private final Serializer serializer;
+ private final byte[] serializedJob;
+
+ BypassJob(Serializer serializer, byte[] serializedJob) {
+ this.serializer = serializer;
+ this.serializedJob = serializedJob;
+ }
+
+ @Override
+ public byte[] call(JobContext jc) throws Exception {
+ Job<?> job = (Job<?>) serializer.deserialize(ByteBuffer.wrap(serializedJob));
+ Object result = job.call(jc);
+ byte[] serializedResult;
+ if (result != null) {
+ ByteBuffer data = serializer.serialize(result);
+ serializedResult = BufferUtils.toByteArray(data);
+ } else {
+ serializedResult = null;
+ }
+ return serializedResult;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/BypassJobWrapper.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/BypassJobWrapper.java b/rsc/src/main/java/org/apache/livy/rsc/driver/BypassJobWrapper.java
new file mode 100644
index 0000000..1fa5bf1
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/BypassJobWrapper.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.driver;
+
+import java.util.List;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobHandle;
+import org.apache.livy.rsc.BypassJobStatus;
+import org.apache.livy.rsc.Utils;
+
+public class BypassJobWrapper extends JobWrapper<byte[]> {
+
+ private volatile byte[] result;
+ private volatile Throwable error;
+ private volatile JobHandle.State state;
+ private volatile List<Integer> newSparkJobs;
+
+ public BypassJobWrapper(RSCDriver driver, String jobId, Job<byte[]> serializedJob) {
+ super(driver, jobId, serializedJob);
+ state = JobHandle.State.QUEUED;
+ }
+
+ @Override
+ public Void call() throws Exception {
+ state = JobHandle.State.STARTED;
+ return super.call();
+ }
+
+ @Override
+ protected synchronized void finished(byte[] result, Throwable error) {
+ if (error == null) {
+ this.result = result;
+ this.state = JobHandle.State.SUCCEEDED;
+ } else {
+ this.error = error;
+ this.state = JobHandle.State.FAILED;
+ }
+ }
+
+ @Override
+ boolean cancel() {
+ if (super.cancel()) {
+ this.state = JobHandle.State.CANCELLED;
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ protected void jobStarted() {
+ // Do nothing; just avoid sending data back to the driver.
+ }
+
+ synchronized BypassJobStatus getStatus() {
+ String stackTrace = error != null ? Utils.stackTraceAsString(error) : null;
+ return new BypassJobStatus(state, result, stackTrace);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/JobContextImpl.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/JobContextImpl.java b/rsc/src/main/java/org/apache/livy/rsc/driver/JobContextImpl.java
new file mode 100644
index 0000000..ddb5713
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/JobContextImpl.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.driver;
+
+import java.io.File;
+import java.lang.reflect.Method;
+
+import org.apache.spark.SparkContext;
+import org.apache.spark.api.java.JavaFutureAction;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.hive.HiveContext;
+import org.apache.spark.streaming.Duration;
+import org.apache.spark.streaming.api.java.JavaStreamingContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.JobContext;
+import org.apache.livy.rsc.Utils;
+
+class JobContextImpl implements JobContext {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JobContextImpl.class);
+
+ private final JavaSparkContext sc;
+ private final File localTmpDir;
+ private volatile SQLContext sqlctx;
+ private volatile HiveContext hivectx;
+ private volatile JavaStreamingContext streamingctx;
+ private final RSCDriver driver;
+ private volatile Object sparksession;
+
+ public JobContextImpl(JavaSparkContext sc, File localTmpDir, RSCDriver driver) {
+ this.sc = sc;
+ this.localTmpDir = localTmpDir;
+ this.driver = driver;
+ }
+
+ @Override
+ public JavaSparkContext sc() {
+ return sc;
+ }
+
+ @Override
+ public Object sparkSession() throws Exception {
+ if (sparksession == null) {
+ synchronized (this) {
+ if (sparksession == null) {
+ try {
+ Class<?> clz = Class.forName("org.apache.spark.sql.SparkSession$");
+ Object spark = clz.getField("MODULE$").get(null);
+ Method m = clz.getMethod("builder");
+ Object builder = m.invoke(spark);
+ builder.getClass().getMethod("sparkContext", SparkContext.class)
+ .invoke(builder, sc.sc());
+ sparksession = builder.getClass().getMethod("getOrCreate").invoke(builder);
+ } catch (Exception e) {
+ LOG.warn("SparkSession is not supported", e);
+ throw e;
+ }
+ }
+ }
+ }
+
+ return sparksession;
+ }
+
+ @Override
+ public SQLContext sqlctx() {
+ if (sqlctx == null) {
+ synchronized (this) {
+ if (sqlctx == null) {
+ sqlctx = new SQLContext(sc);
+ }
+ }
+ }
+ return sqlctx;
+ }
+
+ @Override
+ public HiveContext hivectx() {
+ if (hivectx == null) {
+ synchronized (this) {
+ if (hivectx == null) {
+ hivectx = new HiveContext(sc.sc());
+ }
+ }
+ }
+ return hivectx;
+ }
+
+ @Override
+ public synchronized JavaStreamingContext streamingctx(){
+ Utils.checkState(streamingctx != null, "method createStreamingContext must be called first.");
+ return streamingctx;
+ }
+
+ @Override
+ public synchronized void createStreamingContext(long batchDuration) {
+ Utils.checkState(streamingctx == null, "Streaming context is not null.");
+ streamingctx = new JavaStreamingContext(sc, new Duration(batchDuration));
+ }
+
+ @Override
+ public synchronized void stopStreamingCtx() {
+ Utils.checkState(streamingctx != null, "Streaming Context is null");
+ streamingctx.stop();
+ streamingctx = null;
+ }
+
+ @Override
+ public File getLocalTmpDir() {
+ return localTmpDir;
+ }
+
+ public synchronized void stop() {
+ if (streamingctx != null) {
+ stopStreamingCtx();
+ }
+ if (sc != null) {
+ sc.stop();
+ }
+ }
+
+ public void addFile(String path) {
+ driver.addFile(path);
+ }
+
+ public void addJarOrPyFile(String path) throws Exception {
+ driver.addJarOrPyFile(path);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/JobWrapper.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/JobWrapper.java b/rsc/src/main/java/org/apache/livy/rsc/driver/JobWrapper.java
new file mode 100644
index 0000000..f6df164
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/JobWrapper.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.driver;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.spark.api.java.JavaFutureAction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.livy.Job;
+
+public class JobWrapper<T> implements Callable<Void> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JobWrapper.class);
+
+ public final String jobId;
+
+ private final RSCDriver driver;
+ private final Job<T> job;
+ private final AtomicInteger completed;
+
+ private Future<?> future;
+
+ public JobWrapper(RSCDriver driver, String jobId, Job<T> job) {
+ this.driver = driver;
+ this.jobId = jobId;
+ this.job = job;
+ this.completed = new AtomicInteger();
+ }
+
+ @Override
+ public Void call() throws Exception {
+ try {
+ jobStarted();
+ T result = job.call(driver.jobContext());
+ finished(result, null);
+ } catch (Throwable t) {
+ // Catch throwables in a best-effort to report job status back to the client. It's
+ // re-thrown so that the executor can destroy the affected thread (or the JVM can
+ // die or whatever would happen if the throwable bubbled up).
+ LOG.info("Failed to run job " + jobId, t);
+ finished(null, t);
+ throw new ExecutionException(t);
+ } finally {
+ driver.activeJobs.remove(jobId);
+ }
+ return null;
+ }
+
+ void submit(ExecutorService executor) {
+ this.future = executor.submit(this);
+ }
+
+ void jobDone() {
+ synchronized (completed) {
+ completed.incrementAndGet();
+ completed.notifyAll();
+ }
+ }
+
+ boolean cancel() {
+ return future != null ? future.cancel(true) : true;
+ }
+
+ protected void finished(T result, Throwable error) {
+ if (error == null) {
+ driver.jobFinished(jobId, result, null);
+ } else {
+ driver.jobFinished(jobId, null, error);
+ }
+ }
+
+ protected void jobStarted() {
+ driver.jobStarted(jobId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/org/apache/livy/rsc/driver/MutableClassLoader.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/MutableClassLoader.java b/rsc/src/main/java/org/apache/livy/rsc/driver/MutableClassLoader.java
new file mode 100644
index 0000000..30da79e
--- /dev/null
+++ b/rsc/src/main/java/org/apache/livy/rsc/driver/MutableClassLoader.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.driver;
+
+import java.net.URL;
+import java.net.URLClassLoader;
+
+class MutableClassLoader extends URLClassLoader {
+
+ MutableClassLoader(ClassLoader parent) {
+ super(new URL[] { }, parent);
+ }
+
+ @Override
+ public void addURL(URL url) {
+ super.addURL(url);
+ }
+
+}
[31/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/org/apache/livy/client/common/TestHttpMessages.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/org/apache/livy/client/common/TestHttpMessages.java b/client-common/src/test/java/org/apache/livy/client/common/TestHttpMessages.java
new file mode 100644
index 0000000..53d4107
--- /dev/null
+++ b/client-common/src/test/java/org/apache/livy/client/common/TestHttpMessages.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+import org.apache.livy.JobHandle.State;
+
+public class TestHttpMessages {
+
+ /**
+ * Tests that all defined messages can be serialized and deserialized using Jackson.
+ */
+ @Test
+ public void testMessageSerialization() throws Exception {
+ ObjectMapper mapper = new ObjectMapper();
+
+ for (Class<?> msg : HttpMessages.class.getClasses()) {
+ if (msg.isInterface()) {
+ continue;
+ }
+
+ String name = msg.getSimpleName();
+
+ Constructor c = msg.getConstructors()[0];
+ Object[] params = new Object[c.getParameterTypes().length];
+ Type[] genericTypes = c.getGenericParameterTypes();
+ for (int i = 0; i < params.length; i++) {
+ params[i] = dummyValue(c.getParameterTypes()[i], genericTypes[i]);
+ }
+
+ Object o1 = c.newInstance(params);
+ byte[] serialized = mapper.writeValueAsBytes(o1);
+ Object o2 = mapper.readValue(serialized, msg);
+
+ assertNotNull("could not deserialize " + name, o2);
+ for (Field f : msg.getFields()) {
+ checkEquals(name, f, o1, o2);
+ }
+ }
+
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testJobStatusResultBadState() {
+ new HttpMessages.JobStatus(0L, State.QUEUED, new byte[1], null);
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testJobStatusErrorBadState() {
+ new HttpMessages.JobStatus(0L, State.QUEUED, null, "An Error");
+ }
+
+ private Object dummyValue(Class<?> klass, Type type) {
+ switch (klass.getSimpleName()) {
+ case "int":
+ case "Integer":
+ return 42;
+ case "long": return 84L;
+ case "byte[]": return new byte[] { (byte) 0x42, (byte) 0x84 };
+ case "String": return "test";
+ case "State": return State.SUCCEEDED;
+ case "Map":
+ Map<String, String> map = new HashMap<>();
+ map.put("dummy1", "dummy2");
+ return map;
+ case "List":
+ Class<?> genericType = getGenericArgType(type);
+ return Arrays.asList(dummyValue(genericType, null), dummyValue(genericType, null));
+ default: throw new IllegalArgumentException("FIX ME: " + klass.getSimpleName());
+ }
+ }
+
+ private Class<?> getGenericArgType(Type type) {
+ assertNotNull("FIX ME: null type argument.", type);
+
+ ParameterizedType ptype = (ParameterizedType) type;
+ assertEquals("FIX ME: no support for multiple type arguments.",
+ 1, ptype.getActualTypeArguments().length);
+
+ Type argType = ptype.getActualTypeArguments()[0];
+ assertTrue("FIX ME: type argument is not a class.", argType instanceof Class);
+
+ return (Class<?>) argType;
+ }
+
+ private void checkEquals(String name, Field f, Object o1, Object o2) throws Exception {
+ Object v1 = f.get(o1);
+ Object v2 = f.get(o2);
+
+ boolean match;
+ if (!f.getType().isArray()) {
+ match = v1.equals(v2);
+ } else if (v1 instanceof byte[]) {
+ match = Arrays.equals((byte[]) v1, (byte[]) v2);
+ } else {
+ throw new IllegalArgumentException("FIX ME: " + f.getType().getSimpleName());
+ }
+
+ assertTrue(
+ String.format("Field %s of %s does not match after deserialization.", f.getName(), name),
+ match);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/org/apache/livy/client/common/TestSerializer.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/org/apache/livy/client/common/TestSerializer.java b/client-common/src/test/java/org/apache/livy/client/common/TestSerializer.java
new file mode 100644
index 0000000..65fa555
--- /dev/null
+++ b/client-common/src/test/java/org/apache/livy/client/common/TestSerializer.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestSerializer {
+
+ private static final String MESSAGE = "Hello World!";
+
+ @Test
+ public void testSerializer() throws Exception {
+ Object decoded = doSerDe(MESSAGE);
+ assertEquals(MESSAGE, decoded);
+ }
+
+ @Test
+ public void testUnicodeSerializer() throws Exception {
+ StringBuilder builder = new StringBuilder();
+ for (int x = 0; x < 5000; x++) {
+ builder.append("\u263A");
+ }
+ String testMessage = builder.toString();
+ Object decoded = doSerDe(testMessage);
+ assertEquals(testMessage, decoded);
+ }
+
+ @Test
+ public void testAutoRegistration() throws Exception {
+ Object decoded = doSerDe(new TestMessage(MESSAGE), TestMessage.class);
+ assertTrue(decoded instanceof TestMessage);
+ assertEquals(MESSAGE, ((TestMessage)decoded).data);
+ }
+
+ private Object doSerDe(Object data, Class<?>... klasses) {
+ Serializer s = new Serializer(klasses);
+ ByteBuffer serialized = s.serialize(data);
+ return s.deserialize(serialized);
+ }
+
+ private ByteBuffer newBuffer() {
+ return ByteBuffer.allocate(1024);
+ }
+
+ private static class TestMessage {
+ final String data;
+
+ TestMessage() {
+ this(null);
+ }
+
+ TestMessage(String data) {
+ this.data = data;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/src/test/java/org/apache/livy/client/common/TestTestUtils.java
----------------------------------------------------------------------
diff --git a/client-common/src/test/java/org/apache/livy/client/common/TestTestUtils.java b/client-common/src/test/java/org/apache/livy/client/common/TestTestUtils.java
new file mode 100644
index 0000000..bffb72f
--- /dev/null
+++ b/client-common/src/test/java/org/apache/livy/client/common/TestTestUtils.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.common;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestTestUtils {
+
+ @Test
+ public void testJacocoArgs() {
+ String args1 = TestUtils.getJacocoArgs();
+ String expected1 = System.getProperty("jacoco.args").replace("main.exec", "jacoco-1.exec");
+ assertEquals(expected1, args1);
+
+ String args2 = TestUtils.getJacocoArgs();
+ String expected2 = System.getProperty("jacoco.args").replace("main.exec", "jacoco-2.exec");
+ assertEquals(expected2, args2);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/pom.xml
----------------------------------------------------------------------
diff --git a/client-http/pom.xml b/client-http/pom.xml
index 056d4c0..15e16db 100644
--- a/client-http/pom.xml
+++ b/client-http/pom.xml
@@ -18,37 +18,37 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
</parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-client-http</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<dependencies>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-client-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-test-lib</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-core_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
@@ -80,7 +80,7 @@
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-server</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -126,7 +126,7 @@
<include>*:*</include>
</includes>
<excludes>
- <exclude>com.cloudera.livy:livy-api</exclude>
+ <exclude>org.apache.livy:livy-api</exclude>
</excludes>
</artifactSet>
<filters>
@@ -142,19 +142,19 @@
<relocations>
<relocation>
<pattern>com.esotericsoftware</pattern>
- <shadedPattern>com.cloudera.livy.shaded.kryo</shadedPattern>
+ <shadedPattern>org.apache.livy.shaded.kryo</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>com.cloudera.livy.shaded.jackson</shadedPattern>
+ <shadedPattern>org.apache.livy.shaded.jackson</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons</pattern>
- <shadedPattern>com.cloudera.livy.shaded.apache.commons</shadedPattern>
+ <shadedPattern>org.apache.livy.shaded.apache.commons</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.http</pattern>
- <shadedPattern>com.cloudera.livy.shaded.apache.http</shadedPattern>
+ <shadedPattern>org.apache.livy.shaded.apache.http</shadedPattern>
</relocation>
</relocations>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/java/com/cloudera/livy/client/http/HttpClient.java
----------------------------------------------------------------------
diff --git a/client-http/src/main/java/com/cloudera/livy/client/http/HttpClient.java b/client-http/src/main/java/com/cloudera/livy/client/http/HttpClient.java
deleted file mode 100644
index 943fdb5..0000000
--- a/client-http/src/main/java/com/cloudera/livy/client/http/HttpClient.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.http;
-
-import java.io.File;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobHandle;
-import com.cloudera.livy.LivyClient;
-import com.cloudera.livy.client.common.BufferUtils;
-import com.cloudera.livy.client.common.Serializer;
-import static com.cloudera.livy.client.common.HttpMessages.*;
-import static com.cloudera.livy.client.http.HttpConf.Entry.*;
-
-/**
- * What is currently missing:
- * - monitoring of spark job IDs launched by jobs
- */
-class HttpClient implements LivyClient {
-
- private final HttpConf config;
- private final LivyConnection conn;
- private final int sessionId;
- private final ScheduledExecutorService executor;
- private final Serializer serializer;
-
- private boolean stopped;
-
- HttpClient(URI uri, HttpConf httpConf) {
- this.config = httpConf;
- this.stopped = false;
-
- // If the given URI looks like it refers to an existing session, then try to connect to
- // an existing session. Note this means that any Spark configuration in httpConf will be
- // unused.
- Matcher m = Pattern.compile("(.*)" + LivyConnection.SESSIONS_URI + "/([0-9]+)")
- .matcher(uri.getPath());
-
- try {
- if (m.matches()) {
- URI base = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(),
- m.group(1), uri.getQuery(), uri.getFragment());
-
- this.conn = new LivyConnection(base, httpConf);
- this.sessionId = Integer.parseInt(m.group(2));
- conn.post(null, SessionInfo.class, "/%d/connect", sessionId);
- } else {
- Map<String, String> sessionConf = new HashMap<>();
- for (Map.Entry<String, String> e : config) {
- sessionConf.put(e.getKey(), e.getValue());
- }
-
- ClientMessage create = new CreateClientRequest(sessionConf);
- this.conn = new LivyConnection(uri, httpConf);
- this.sessionId = conn.post(create, SessionInfo.class, "/").id;
- }
- } catch (Exception e) {
- throw propagate(e);
- }
-
- // Because we only have one connection to the server, we don't need more than a single
- // threaded executor here.
- this.executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new Thread(r, "HttpClient-" + sessionId);
- t.setDaemon(true);
- return t;
- }
- });
-
- this.serializer = new Serializer();
- }
-
- @Override
- public <T> JobHandle<T> submit(Job<T> job) {
- return sendJob("submit-job", job);
- }
-
- @Override
- public <T> Future<T> run(Job<T> job) {
- return sendJob("run-job", job);
- }
-
- @Override
- public synchronized void stop(boolean shutdownContext) {
- if (!stopped) {
- executor.shutdownNow();
- try {
- if (shutdownContext) {
- conn.delete(Map.class, "/%s", sessionId);
- }
- } catch (Exception e) {
- throw propagate(e);
- } finally {
- try {
- conn.close();
- } catch (Exception e) {
- // Ignore.
- }
- }
- stopped = true;
- }
- }
-
- public Future<?> uploadJar(File jar) {
- return uploadResource(jar, "upload-jar", "jar");
- }
-
- @Override
- public Future<?> addJar(URI uri) {
- return addResource("add-jar", uri);
- }
-
- public Future<?> uploadFile(File file) {
- return uploadResource(file, "upload-file", "file");
- }
-
- @Override
- public Future<?> addFile(URI uri) {
- return addResource("add-file", uri);
- }
-
- private Future<?> uploadResource(final File file, final String command, final String paramName) {
- Callable<Void> task = new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- conn.post(file, Void.class, paramName, "/%d/%s", sessionId, command);
- return null;
- }
- };
- return executor.submit(task);
- }
-
- private Future<?> addResource(final String command, final URI resource) {
- Callable<Void> task = new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- ClientMessage msg = new AddResource(resource.toString());
- conn.post(msg, Void.class, "/%d/%s", sessionId, command);
- return null;
- }
- };
- return executor.submit(task);
- }
-
- private <T> JobHandleImpl<T> sendJob(final String command, Job<T> job) {
- final ByteBuffer serializedJob = serializer.serialize(job);
- JobHandleImpl<T> handle = new JobHandleImpl<T>(config, conn, sessionId, executor, serializer);
- handle.start(command, serializedJob);
- return handle;
- }
-
- private RuntimeException propagate(Exception cause) {
- if (cause instanceof RuntimeException) {
- throw (RuntimeException) cause;
- } else {
- throw new RuntimeException(cause);
- }
- }
-
- // For testing.
- int getSessionId() {
- return sessionId;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/java/com/cloudera/livy/client/http/HttpClientFactory.java
----------------------------------------------------------------------
diff --git a/client-http/src/main/java/com/cloudera/livy/client/http/HttpClientFactory.java b/client-http/src/main/java/com/cloudera/livy/client/http/HttpClientFactory.java
deleted file mode 100644
index 2062ef2..0000000
--- a/client-http/src/main/java/com/cloudera/livy/client/http/HttpClientFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.http;
-
-import java.net.URI;
-import java.util.Properties;
-
-import com.cloudera.livy.LivyClient;
-import com.cloudera.livy.LivyClientFactory;
-
-/**
- * Factory for HTTP Livy clients.
- */
-public final class HttpClientFactory implements LivyClientFactory {
-
- @Override
- public LivyClient createClient(URI uri, Properties config) {
- if (!"http".equals(uri.getScheme()) && !"https".equals(uri.getScheme())) {
- return null;
- }
-
- return new HttpClient(uri, new HttpConf(config));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/java/com/cloudera/livy/client/http/HttpConf.java
----------------------------------------------------------------------
diff --git a/client-http/src/main/java/com/cloudera/livy/client/http/HttpConf.java b/client-http/src/main/java/com/cloudera/livy/client/http/HttpConf.java
deleted file mode 100644
index 4ca3665..0000000
--- a/client-http/src/main/java/com/cloudera/livy/client/http/HttpConf.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.http;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import com.cloudera.livy.client.common.ClientConf;
-
-class HttpConf extends ClientConf<HttpConf> {
-
- private static final String HTTP_CONF_PREFIX = "livy.client.http.";
-
- static enum Entry implements ConfEntry {
- CONNECTION_TIMEOUT("connection.timeout", "10s"),
- CONNECTION_IDLE_TIMEOUT("connection.idle.timeout", "10m"),
- SOCKET_TIMEOUT("connection.socket.timeout", "5m"),
-
- JOB_INITIAL_POLL_INTERVAL("job.initial-poll-interval", "100ms"),
- JOB_MAX_POLL_INTERVAL("job.max-poll-interval", "5s"),
-
- CONTENT_COMPRESS_ENABLE("content.compress.enable", true),
-
- // Kerberos related configuration
- SPNEGO_ENABLED("spnego.enable", false),
- AUTH_LOGIN_CONFIG("auth.login.config", null),
- KRB5_DEBUG_ENABLED("krb5.debug", false),
- KRB5_CONF("krb5.conf", null);
-
- private final String key;
- private final Object dflt;
-
- private Entry(String key, Object dflt) {
- this.key = HTTP_CONF_PREFIX + key;
- this.dflt = dflt;
- }
-
- @Override
- public String key() { return key; }
-
- @Override
- public Object dflt() { return dflt; }
- }
-
- HttpConf(Properties config) {
- super(config);
-
- if (getBoolean(Entry.SPNEGO_ENABLED)) {
- if (get(Entry.AUTH_LOGIN_CONFIG ) == null) {
- throw new IllegalArgumentException(Entry.AUTH_LOGIN_CONFIG.key + " should not be null");
- }
-
- if (get(Entry.KRB5_CONF) == null) {
- throw new IllegalArgumentException(Entry.KRB5_CONF.key + " should not be null");
- }
-
- System.setProperty("java.security.auth.login.config", get(Entry.AUTH_LOGIN_CONFIG));
- System.setProperty("java.security.krb5.conf", get(Entry.KRB5_CONF));
- System.setProperty(
- "sun.security.krb5.debug", String.valueOf(getBoolean(Entry.KRB5_DEBUG_ENABLED)));
- // This is needed to get Kerberos credentials from the environment, instead of
- // requiring the application to manually obtain the credentials.
- System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
- }
- }
-
- boolean isSpnegoEnabled() {
- return getBoolean(Entry.SPNEGO_ENABLED);
- }
-
- private static final Map<String, DeprecatedConf> configsWithAlternatives
- = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{
- put(HttpConf.Entry.JOB_INITIAL_POLL_INTERVAL.key, DepConf.JOB_INITIAL_POLL_INTERVAL);
- put(HttpConf.Entry.JOB_MAX_POLL_INTERVAL.key, DepConf.JOB_MAX_POLL_INTERVAL);
- }});
-
- // Maps deprecated key to DeprecatedConf with the same key.
- // There are no deprecated configs without alternatives currently.
- private static final Map<String, DeprecatedConf> deprecatedConfigs
- = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>());
-
- protected Map<String, DeprecatedConf> getConfigsWithAlternatives() {
- return configsWithAlternatives;
- }
-
- protected Map<String, DeprecatedConf> getDeprecatedConfigs() {
- return deprecatedConfigs;
- }
-
- static enum DepConf implements DeprecatedConf {
- JOB_INITIAL_POLL_INTERVAL("job.initial_poll_interval", "0.4"),
- JOB_MAX_POLL_INTERVAL("job.max_poll_interval", "0.4");
-
- private final String key;
- private final String version;
- private final String deprecationMessage;
-
- private DepConf(String key, String version) {
- this(key, version, "");
- }
-
- private DepConf(String key, String version, String deprecationMessage) {
- this.key = HTTP_CONF_PREFIX + key;
- this.version = version;
- this.deprecationMessage = deprecationMessage;
- }
-
- @Override
- public String key() { return key; }
-
- @Override
- public String version() { return version; }
-
- @Override
- public String deprecationMessage() { return deprecationMessage; }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/java/com/cloudera/livy/client/http/JobHandleImpl.java
----------------------------------------------------------------------
diff --git a/client-http/src/main/java/com/cloudera/livy/client/http/JobHandleImpl.java b/client-http/src/main/java/com/cloudera/livy/client/http/JobHandleImpl.java
deleted file mode 100644
index e437059..0000000
--- a/client-http/src/main/java/com/cloudera/livy/client/http/JobHandleImpl.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.http;
-
-import java.nio.ByteBuffer;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import com.cloudera.livy.JobHandle;
-import com.cloudera.livy.client.common.AbstractJobHandle;
-import com.cloudera.livy.client.common.BufferUtils;
-import com.cloudera.livy.client.common.Serializer;
-import static com.cloudera.livy.client.common.HttpMessages.*;
-import static com.cloudera.livy.client.http.HttpConf.Entry.*;
-
-class JobHandleImpl<T> extends AbstractJobHandle<T> {
-
- private final long sessionId;
- private final LivyConnection conn;
- private final ScheduledExecutorService executor;
- private final Object lock;
- private final Serializer serializer;
-
- private final long initialPollInterval;
- private final long maxPollInterval;
-
- private long jobId;
- private T result;
- private Throwable error;
- private volatile boolean isDone;
- private volatile boolean isCancelled;
- private volatile boolean isCancelPending;
- private volatile ScheduledFuture<?> pollTask;
-
- JobHandleImpl(
- HttpConf config,
- LivyConnection conn,
- long sessionId,
- ScheduledExecutorService executor,
- Serializer s) {
- this.conn = conn;
- this.sessionId = sessionId;
- this.executor = executor;
- this.lock = new Object();
- this.serializer = s;
- this.isDone = false;
-
- this.initialPollInterval = config.getTimeAsMs(JOB_INITIAL_POLL_INTERVAL);
- this.maxPollInterval = config.getTimeAsMs(JOB_MAX_POLL_INTERVAL);
-
- if (initialPollInterval <= 0) {
- throw new IllegalArgumentException("Invalid initial poll interval.");
- }
- if (maxPollInterval <= 0 || maxPollInterval < initialPollInterval) {
- throw new IllegalArgumentException(
- "Invalid max poll interval, or lower than initial interval.");
- }
-
- // The job ID is set asynchronously, and there might be a call to cancel() before it's
- // set. So cancel() will always set the isCancelPending flag, even if there's no job
- // ID yet. If the thread setting the job ID sees that flag, it will send a cancel request
- // to the server. There's still a possibility that two cancel requests will be sent,
- // but that doesn't cause any harm.
- this.isCancelPending = false;
- this.jobId = -1;
- }
-
- @Override
- public T get() throws ExecutionException, InterruptedException {
- try {
- return get(true, -1, TimeUnit.MILLISECONDS);
- } catch (TimeoutException te) {
- // Not gonna happen.
- throw new RuntimeException(te);
- }
- }
-
- @Override
- public T get(long timeout, TimeUnit unit)
- throws ExecutionException, InterruptedException, TimeoutException {
- return get(false, timeout, unit);
- }
-
- @Override
- public boolean isDone() {
- return isDone;
- }
-
- @Override
- public boolean isCancelled() {
- return isCancelled;
- }
-
- @Override
- public boolean cancel(final boolean mayInterrupt) {
- // Do a best-effort to detect if already cancelled, but the final say is always
- // on the server side. Don't block the caller, though.
- if (!isCancelled && !isCancelPending) {
- isCancelPending = true;
- if (jobId > -1) {
- sendCancelRequest(jobId);
- }
- return true;
- }
-
- return false;
- }
-
- @Override
- protected T result() {
- return result;
- }
-
- @Override
- protected Throwable error() {
- return error;
- }
-
- void start(final String command, final ByteBuffer serializedJob) {
- Runnable task = new Runnable() {
- @Override
- public void run() {
- try {
- ClientMessage msg = new SerializedJob(BufferUtils.toByteArray(serializedJob));
- JobStatus status = conn.post(msg, JobStatus.class, "/%d/%s", sessionId, command);
-
- if (isCancelPending) {
- sendCancelRequest(status.id);
- }
-
- jobId = status.id;
-
- pollTask = executor.schedule(new JobPollTask(initialPollInterval),
- initialPollInterval, TimeUnit.MILLISECONDS);
- } catch (Exception e) {
- setResult(null, e, State.FAILED);
- }
- }
- };
- executor.submit(task);
- }
-
- private void sendCancelRequest(final long id) {
- executor.submit(new Runnable() {
- @Override
- public void run() {
- try {
- conn.post(null, Void.class, "/%d/jobs/%d/cancel", sessionId, id);
- } catch (Exception e) {
- setResult(null, e, State.FAILED);
- }
- }
- });
- }
-
- private T get(boolean waitIndefinitely, long timeout, TimeUnit unit)
- throws ExecutionException, InterruptedException, TimeoutException {
- if (!isDone) {
- synchronized (lock) {
- if (waitIndefinitely) {
- while (!isDone) {
- lock.wait();
- }
- } else {
- long now = System.nanoTime();
- long deadline = now + unit.toNanos(timeout);
- while (!isDone && deadline > now) {
- lock.wait(TimeUnit.NANOSECONDS.toMillis(deadline - now));
- now = System.nanoTime();
- }
- if (!isDone) {
- throw new TimeoutException();
- }
- }
- }
- }
- if (isCancelled) {
- throw new CancellationException();
- }
- if (error != null) {
- throw new ExecutionException(error);
- }
- return result;
- }
-
- private void setResult(T result, Throwable error, State newState) {
- if (!isDone) {
- synchronized (lock) {
- if (!isDone) {
- this.result = result;
- this.error = error;
- this.isDone = true;
- changeState(newState);
- }
- lock.notifyAll();
- }
- }
- }
-
- private class JobPollTask implements Runnable {
-
- private long currentInterval;
-
- JobPollTask(long currentInterval) {
- this.currentInterval = currentInterval;
- }
-
- @Override
- public void run() {
- try {
- JobStatus status = conn.get(JobStatus.class, "/%d/jobs/%d", sessionId, jobId);
- T result = null;
- Throwable error = null;
- boolean finished = false;
-
- switch (status.state) {
- case SUCCEEDED:
- if (status.result != null) {
- @SuppressWarnings("unchecked")
- T localResult = (T) serializer.deserialize(ByteBuffer.wrap(status.result));
- result = localResult;
- }
- finished = true;
- break;
-
- case FAILED:
- // TODO: better exception.
- error = new RuntimeException(status.error);
- finished = true;
- break;
-
- case CANCELLED:
- isCancelled = true;
- finished = true;
- break;
-
- default:
- // Nothing to do.
- }
- if (finished) {
- setResult(result, error, status.state);
- } else if (status.state != state) {
- changeState(status.state);
- }
- if (!finished) {
- currentInterval = Math.min(currentInterval * 2, maxPollInterval);
- pollTask = executor.schedule(this, currentInterval, TimeUnit.MILLISECONDS);
- }
- } catch (Exception e) {
- setResult(null, e, State.FAILED);
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/java/com/cloudera/livy/client/http/LivyConnection.java
----------------------------------------------------------------------
diff --git a/client-http/src/main/java/com/cloudera/livy/client/http/LivyConnection.java b/client-http/src/main/java/com/cloudera/livy/client/http/LivyConnection.java
deleted file mode 100644
index e48013f..0000000
--- a/client-http/src/main/java/com/cloudera/livy/client/http/LivyConnection.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.http;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URLDecoder;
-import java.nio.charset.StandardCharsets;
-import java.security.Principal;
-import java.util.concurrent.TimeUnit;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHeaders;
-import org.apache.http.HttpStatus;
-import org.apache.http.auth.AuthSchemeProvider;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.Credentials;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.config.AuthSchemes;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.*;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.config.Registry;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.entity.mime.MultipartEntityBuilder;
-import org.apache.http.entity.mime.content.FileBody;
-import org.apache.http.impl.NoConnectionReuseStrategy;
-import org.apache.http.impl.auth.SPNegoSchemeFactory;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
-import org.apache.http.util.EntityUtils;
-
-import static com.cloudera.livy.client.http.HttpConf.Entry.*;
-
-/**
- * Abstracts a connection to the Livy server; serializes multiple requests so that we only need
- * one active HTTP connection (to keep resource usage down).
- */
-class LivyConnection {
-
- static final String SESSIONS_URI = "/sessions";
- private static final String APPLICATION_JSON = "application/json";
-
- private final URI server;
- private final String uriRoot;
- private final CloseableHttpClient client;
- private final ObjectMapper mapper;
-
- LivyConnection(URI uri, final HttpConf config) {
- HttpClientContext ctx = HttpClientContext.create();
- int port = uri.getPort() > 0 ? uri.getPort() : 8998;
-
- String path = uri.getPath() != null ? uri.getPath() : "";
- this.uriRoot = path + SESSIONS_URI;
-
- RequestConfig reqConfig = new RequestConfig() {
- @Override
- public int getConnectTimeout() {
- return (int) config.getTimeAsMs(CONNECTION_TIMEOUT);
- }
-
- @Override
- public int getSocketTimeout() {
- return (int) config.getTimeAsMs(SOCKET_TIMEOUT);
- }
-
- @Override
- public boolean isAuthenticationEnabled() {
- return true;
- }
-
- @Override
- public boolean isContentCompressionEnabled() {
- return config.getBoolean(CONTENT_COMPRESS_ENABLE);
- }
- };
-
- Credentials credentials;
- // If user info is specified in the url, pass them to the CredentialsProvider.
- if (uri.getUserInfo() != null) {
- String[] userInfo = uri.getUserInfo().split(":");
- if (userInfo.length < 1) {
- throw new IllegalArgumentException("Malformed user info in the url.");
- }
- try {
- String username = URLDecoder.decode(userInfo[0], StandardCharsets.UTF_8.name());
- String password = "";
- if (userInfo.length > 1) {
- password = URLDecoder.decode(userInfo[1], StandardCharsets.UTF_8.name());
- }
- credentials = new UsernamePasswordCredentials(username, password);
- } catch (Exception e) {
- throw new IllegalArgumentException("User info in the url contains bad characters.", e);
- }
- } else {
- credentials = new Credentials() {
- @Override
- public String getPassword() {
- return null;
- }
-
- @Override
- public Principal getUserPrincipal() {
- return null;
- }
- };
- }
-
- CredentialsProvider credsProvider = new BasicCredentialsProvider();
- credsProvider.setCredentials(AuthScope.ANY, credentials);
-
- HttpClientBuilder builder = HttpClientBuilder.create()
- .disableAutomaticRetries()
- .evictExpiredConnections()
- .evictIdleConnections(config.getTimeAsMs(CONNECTION_IDLE_TIMEOUT), TimeUnit.MILLISECONDS)
- .setConnectionManager(new BasicHttpClientConnectionManager())
- .setConnectionReuseStrategy(new NoConnectionReuseStrategy())
- .setDefaultRequestConfig(reqConfig)
- .setMaxConnTotal(1)
- .setDefaultCredentialsProvider(credsProvider)
- .setUserAgent("livy-client-http");
-
- if (config.isSpnegoEnabled()) {
- Registry<AuthSchemeProvider> authSchemeProviderRegistry =
- RegistryBuilder.<AuthSchemeProvider>create()
- .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
- .build();
- builder.setDefaultAuthSchemeRegistry(authSchemeProviderRegistry);
- }
-
- this.server = uri;
- this.client = builder.build();
- this.mapper = new ObjectMapper();
- }
-
- synchronized void close() throws IOException {
- client.close();
- }
-
- synchronized <V> V delete(Class<V> retType, String uri, Object... uriParams) throws Exception {
- return sendJSONRequest(new HttpDelete(), retType, uri, uriParams);
- }
-
- synchronized <V> V get(Class<V> retType, String uri, Object... uriParams) throws Exception {
- return sendJSONRequest(new HttpGet(), retType, uri, uriParams);
- }
-
- synchronized <V> V post(
- Object body,
- Class<V> retType,
- String uri,
- Object... uriParams) throws Exception {
- HttpPost post = new HttpPost();
- if (body != null) {
- byte[] bodyBytes = mapper.writeValueAsBytes(body);
- post.setEntity(new ByteArrayEntity(bodyBytes));
- }
- return sendJSONRequest(post, retType, uri, uriParams);
- }
-
- synchronized <V> V post(
- File f,
- Class<V> retType,
- String paramName,
- String uri,
- Object... uriParams) throws Exception {
- HttpPost post = new HttpPost();
- MultipartEntityBuilder builder = MultipartEntityBuilder.create();
- builder.addPart(paramName, new FileBody(f));
- post.setEntity(builder.build());
- return sendRequest(post, retType, uri, uriParams);
- }
-
- private <V> V sendJSONRequest(
- HttpRequestBase req,
- Class<V> retType,
- String uri,
- Object... uriParams) throws Exception {
- req.setHeader(HttpHeaders.ACCEPT, APPLICATION_JSON);
- req.setHeader(HttpHeaders.CONTENT_TYPE, APPLICATION_JSON);
- req.setHeader(HttpHeaders.CONTENT_ENCODING, "UTF-8");
- return sendRequest(req, retType, uri, uriParams);
- }
-
- private <V> V sendRequest(
- HttpRequestBase req,
- Class<V> retType,
- String uri,
- Object... uriParams) throws Exception {
- req.setURI(new URI(server.getScheme(), null, server.getHost(), server.getPort(),
- uriRoot + String.format(uri, uriParams), null, null));
- // It is no harm to set X-Requested-By when csrf protection is disabled.
- if (req instanceof HttpPost || req instanceof HttpDelete || req instanceof HttpPut
- || req instanceof HttpPatch) {
- req.addHeader("X-Requested-By", "livy");
- }
- try (CloseableHttpResponse res = client.execute(req)) {
- int status = (res.getStatusLine().getStatusCode() / 100) * 100;
- HttpEntity entity = res.getEntity();
- if (status == HttpStatus.SC_OK) {
- if (!Void.class.equals(retType)) {
- return mapper.readValue(entity.getContent(), retType);
- } else {
- return null;
- }
- } else {
- String error = EntityUtils.toString(entity);
- throw new IOException(String.format("%s: %s", res.getStatusLine().getReasonPhrase(),
- error));
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/java/org/apache/livy/client/http/HttpClient.java
----------------------------------------------------------------------
diff --git a/client-http/src/main/java/org/apache/livy/client/http/HttpClient.java b/client-http/src/main/java/org/apache/livy/client/http/HttpClient.java
new file mode 100644
index 0000000..f40148f
--- /dev/null
+++ b/client-http/src/main/java/org/apache/livy/client/http/HttpClient.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.http;
+
+import java.io.File;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobHandle;
+import org.apache.livy.LivyClient;
+import org.apache.livy.client.common.Serializer;
+import static org.apache.livy.client.common.HttpMessages.*;
+
+/**
+ * What is currently missing:
+ * - monitoring of spark job IDs launched by jobs
+ */
+class HttpClient implements LivyClient {
+
+ private final HttpConf config;
+ private final LivyConnection conn;
+ private final int sessionId;
+ private final ScheduledExecutorService executor;
+ private final Serializer serializer;
+
+ private boolean stopped;
+
+ HttpClient(URI uri, HttpConf httpConf) {
+ this.config = httpConf;
+ this.stopped = false;
+
+ // If the given URI looks like it refers to an existing session, then try to connect to
+ // an existing session. Note this means that any Spark configuration in httpConf will be
+ // unused.
+ Matcher m = Pattern.compile("(.*)" + LivyConnection.SESSIONS_URI + "/([0-9]+)")
+ .matcher(uri.getPath());
+
+ try {
+ if (m.matches()) {
+ URI base = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(),
+ m.group(1), uri.getQuery(), uri.getFragment());
+
+ this.conn = new LivyConnection(base, httpConf);
+ this.sessionId = Integer.parseInt(m.group(2));
+ conn.post(null, SessionInfo.class, "/%d/connect", sessionId);
+ } else {
+ Map<String, String> sessionConf = new HashMap<>();
+ for (Map.Entry<String, String> e : config) {
+ sessionConf.put(e.getKey(), e.getValue());
+ }
+
+ ClientMessage create = new CreateClientRequest(sessionConf);
+ this.conn = new LivyConnection(uri, httpConf);
+ this.sessionId = conn.post(create, SessionInfo.class, "/").id;
+ }
+ } catch (Exception e) {
+ throw propagate(e);
+ }
+
+ // Because we only have one connection to the server, we don't need more than a single
+ // threaded executor here.
+ this.executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r, "HttpClient-" + sessionId);
+ t.setDaemon(true);
+ return t;
+ }
+ });
+
+ this.serializer = new Serializer();
+ }
+
+ @Override
+ public <T> JobHandle<T> submit(Job<T> job) {
+ return sendJob("submit-job", job);
+ }
+
+ @Override
+ public <T> Future<T> run(Job<T> job) {
+ return sendJob("run-job", job);
+ }
+
+ @Override
+ public synchronized void stop(boolean shutdownContext) {
+ if (!stopped) {
+ executor.shutdownNow();
+ try {
+ if (shutdownContext) {
+ conn.delete(Map.class, "/%s", sessionId);
+ }
+ } catch (Exception e) {
+ throw propagate(e);
+ } finally {
+ try {
+ conn.close();
+ } catch (Exception e) {
+ // Ignore.
+ }
+ }
+ stopped = true;
+ }
+ }
+
+ public Future<?> uploadJar(File jar) {
+ return uploadResource(jar, "upload-jar", "jar");
+ }
+
+ @Override
+ public Future<?> addJar(URI uri) {
+ return addResource("add-jar", uri);
+ }
+
+ public Future<?> uploadFile(File file) {
+ return uploadResource(file, "upload-file", "file");
+ }
+
+ @Override
+ public Future<?> addFile(URI uri) {
+ return addResource("add-file", uri);
+ }
+
+ private Future<?> uploadResource(final File file, final String command, final String paramName) {
+ Callable<Void> task = new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ conn.post(file, Void.class, paramName, "/%d/%s", sessionId, command);
+ return null;
+ }
+ };
+ return executor.submit(task);
+ }
+
+ private Future<?> addResource(final String command, final URI resource) {
+ Callable<Void> task = new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ ClientMessage msg = new AddResource(resource.toString());
+ conn.post(msg, Void.class, "/%d/%s", sessionId, command);
+ return null;
+ }
+ };
+ return executor.submit(task);
+ }
+
+ private <T> JobHandleImpl<T> sendJob(final String command, Job<T> job) {
+ final ByteBuffer serializedJob = serializer.serialize(job);
+ JobHandleImpl<T> handle = new JobHandleImpl<T>(config, conn, sessionId, executor, serializer);
+ handle.start(command, serializedJob);
+ return handle;
+ }
+
+ private RuntimeException propagate(Exception cause) {
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ } else {
+ throw new RuntimeException(cause);
+ }
+ }
+
+ // For testing.
+ int getSessionId() {
+ return sessionId;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/java/org/apache/livy/client/http/HttpClientFactory.java
----------------------------------------------------------------------
diff --git a/client-http/src/main/java/org/apache/livy/client/http/HttpClientFactory.java b/client-http/src/main/java/org/apache/livy/client/http/HttpClientFactory.java
new file mode 100644
index 0000000..622a74e
--- /dev/null
+++ b/client-http/src/main/java/org/apache/livy/client/http/HttpClientFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.http;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.apache.livy.LivyClient;
+import org.apache.livy.LivyClientFactory;
+
+/**
+ * Factory for HTTP Livy clients.
+ */
+public final class HttpClientFactory implements LivyClientFactory {
+
+ @Override
+ public LivyClient createClient(URI uri, Properties config) {
+ if (!"http".equals(uri.getScheme()) && !"https".equals(uri.getScheme())) {
+ return null;
+ }
+
+ return new HttpClient(uri, new HttpConf(config));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/java/org/apache/livy/client/http/HttpConf.java
----------------------------------------------------------------------
diff --git a/client-http/src/main/java/org/apache/livy/client/http/HttpConf.java b/client-http/src/main/java/org/apache/livy/client/http/HttpConf.java
new file mode 100644
index 0000000..2ae7475
--- /dev/null
+++ b/client-http/src/main/java/org/apache/livy/client/http/HttpConf.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.http;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.livy.client.common.ClientConf;
+
+class HttpConf extends ClientConf<HttpConf> {
+
+ private static final String HTTP_CONF_PREFIX = "livy.client.http.";
+
+ static enum Entry implements ConfEntry {
+ CONNECTION_TIMEOUT("connection.timeout", "10s"),
+ CONNECTION_IDLE_TIMEOUT("connection.idle.timeout", "10m"),
+ SOCKET_TIMEOUT("connection.socket.timeout", "5m"),
+
+ JOB_INITIAL_POLL_INTERVAL("job.initial-poll-interval", "100ms"),
+ JOB_MAX_POLL_INTERVAL("job.max-poll-interval", "5s"),
+
+ CONTENT_COMPRESS_ENABLE("content.compress.enable", true),
+
+ // Kerberos related configuration
+ SPNEGO_ENABLED("spnego.enable", false),
+ AUTH_LOGIN_CONFIG("auth.login.config", null),
+ KRB5_DEBUG_ENABLED("krb5.debug", false),
+ KRB5_CONF("krb5.conf", null);
+
+ private final String key;
+ private final Object dflt;
+
+ private Entry(String key, Object dflt) {
+ this.key = HTTP_CONF_PREFIX + key;
+ this.dflt = dflt;
+ }
+
+ @Override
+ public String key() { return key; }
+
+ @Override
+ public Object dflt() { return dflt; }
+ }
+
+ HttpConf(Properties config) {
+ super(config);
+
+ if (getBoolean(Entry.SPNEGO_ENABLED)) {
+ if (get(Entry.AUTH_LOGIN_CONFIG ) == null) {
+ throw new IllegalArgumentException(Entry.AUTH_LOGIN_CONFIG.key + " should not be null");
+ }
+
+ if (get(Entry.KRB5_CONF) == null) {
+ throw new IllegalArgumentException(Entry.KRB5_CONF.key + " should not be null");
+ }
+
+ System.setProperty("java.security.auth.login.config", get(Entry.AUTH_LOGIN_CONFIG));
+ System.setProperty("java.security.krb5.conf", get(Entry.KRB5_CONF));
+ System.setProperty(
+ "sun.security.krb5.debug", String.valueOf(getBoolean(Entry.KRB5_DEBUG_ENABLED)));
+ // This is needed to get Kerberos credentials from the environment, instead of
+ // requiring the application to manually obtain the credentials.
+ System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
+ }
+ }
+
+ boolean isSpnegoEnabled() {
+ return getBoolean(Entry.SPNEGO_ENABLED);
+ }
+
+ private static final Map<String, DeprecatedConf> configsWithAlternatives
+ = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{
+ put(HttpConf.Entry.JOB_INITIAL_POLL_INTERVAL.key, DepConf.JOB_INITIAL_POLL_INTERVAL);
+ put(HttpConf.Entry.JOB_MAX_POLL_INTERVAL.key, DepConf.JOB_MAX_POLL_INTERVAL);
+ }});
+
+ // Maps deprecated key to DeprecatedConf with the same key.
+ // There are no deprecated configs without alternatives currently.
+ private static final Map<String, DeprecatedConf> deprecatedConfigs
+ = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>());
+
+ protected Map<String, DeprecatedConf> getConfigsWithAlternatives() {
+ return configsWithAlternatives;
+ }
+
+ protected Map<String, DeprecatedConf> getDeprecatedConfigs() {
+ return deprecatedConfigs;
+ }
+
+ static enum DepConf implements DeprecatedConf {
+ JOB_INITIAL_POLL_INTERVAL("job.initial_poll_interval", "0.4"),
+ JOB_MAX_POLL_INTERVAL("job.max_poll_interval", "0.4");
+
+ private final String key;
+ private final String version;
+ private final String deprecationMessage;
+
+ private DepConf(String key, String version) {
+ this(key, version, "");
+ }
+
+ private DepConf(String key, String version, String deprecationMessage) {
+ this.key = HTTP_CONF_PREFIX + key;
+ this.version = version;
+ this.deprecationMessage = deprecationMessage;
+ }
+
+ @Override
+ public String key() { return key; }
+
+ @Override
+ public String version() { return version; }
+
+ @Override
+ public String deprecationMessage() { return deprecationMessage; }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/java/org/apache/livy/client/http/JobHandleImpl.java
----------------------------------------------------------------------
diff --git a/client-http/src/main/java/org/apache/livy/client/http/JobHandleImpl.java b/client-http/src/main/java/org/apache/livy/client/http/JobHandleImpl.java
new file mode 100644
index 0000000..f0ffb59
--- /dev/null
+++ b/client-http/src/main/java/org/apache/livy/client/http/JobHandleImpl.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.http;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.livy.client.common.AbstractJobHandle;
+import org.apache.livy.client.common.BufferUtils;
+import org.apache.livy.client.common.Serializer;
+import static org.apache.livy.client.common.HttpMessages.*;
+
+class JobHandleImpl<T> extends AbstractJobHandle<T> {
+
+ private final long sessionId;
+ private final LivyConnection conn;
+ private final ScheduledExecutorService executor;
+ private final Object lock;
+ private final Serializer serializer;
+
+ private final long initialPollInterval;
+ private final long maxPollInterval;
+
+ private long jobId;
+ private T result;
+ private Throwable error;
+ private volatile boolean isDone;
+ private volatile boolean isCancelled;
+ private volatile boolean isCancelPending;
+ private volatile ScheduledFuture<?> pollTask;
+
+ JobHandleImpl(
+ HttpConf config,
+ LivyConnection conn,
+ long sessionId,
+ ScheduledExecutorService executor,
+ Serializer s) {
+ this.conn = conn;
+ this.sessionId = sessionId;
+ this.executor = executor;
+ this.lock = new Object();
+ this.serializer = s;
+ this.isDone = false;
+
+ this.initialPollInterval = config.getTimeAsMs(HttpConf.Entry.JOB_INITIAL_POLL_INTERVAL);
+ this.maxPollInterval = config.getTimeAsMs(HttpConf.Entry.JOB_MAX_POLL_INTERVAL);
+
+ if (initialPollInterval <= 0) {
+ throw new IllegalArgumentException("Invalid initial poll interval.");
+ }
+ if (maxPollInterval <= 0 || maxPollInterval < initialPollInterval) {
+ throw new IllegalArgumentException(
+ "Invalid max poll interval, or lower than initial interval.");
+ }
+
+ // The job ID is set asynchronously, and there might be a call to cancel() before it's
+ // set. So cancel() will always set the isCancelPending flag, even if there's no job
+ // ID yet. If the thread setting the job ID sees that flag, it will send a cancel request
+ // to the server. There's still a possibility that two cancel requests will be sent,
+ // but that doesn't cause any harm.
+ this.isCancelPending = false;
+ this.jobId = -1;
+ }
+
+ @Override
+ public T get() throws ExecutionException, InterruptedException {
+ try {
+ return get(true, -1, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException te) {
+ // Not gonna happen.
+ throw new RuntimeException(te);
+ }
+ }
+
+ @Override
+ public T get(long timeout, TimeUnit unit)
+ throws ExecutionException, InterruptedException, TimeoutException {
+ return get(false, timeout, unit);
+ }
+
+ @Override
+ public boolean isDone() {
+ return isDone;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return isCancelled;
+ }
+
+ @Override
+ public boolean cancel(final boolean mayInterrupt) {
+ // Do a best-effort to detect if already cancelled, but the final say is always
+ // on the server side. Don't block the caller, though.
+ if (!isCancelled && !isCancelPending) {
+ isCancelPending = true;
+ if (jobId > -1) {
+ sendCancelRequest(jobId);
+ }
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ protected T result() {
+ return result;
+ }
+
+ @Override
+ protected Throwable error() {
+ return error;
+ }
+
+ void start(final String command, final ByteBuffer serializedJob) {
+ Runnable task = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ ClientMessage msg = new SerializedJob(BufferUtils.toByteArray(serializedJob));
+ JobStatus status = conn.post(msg, JobStatus.class, "/%d/%s", sessionId, command);
+
+ if (isCancelPending) {
+ sendCancelRequest(status.id);
+ }
+
+ jobId = status.id;
+
+ pollTask = executor.schedule(new JobPollTask(initialPollInterval),
+ initialPollInterval, TimeUnit.MILLISECONDS);
+ } catch (Exception e) {
+ setResult(null, e, State.FAILED);
+ }
+ }
+ };
+ executor.submit(task);
+ }
+
+ private void sendCancelRequest(final long id) {
+ executor.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ conn.post(null, Void.class, "/%d/jobs/%d/cancel", sessionId, id);
+ } catch (Exception e) {
+ setResult(null, e, State.FAILED);
+ }
+ }
+ });
+ }
+
+ private T get(boolean waitIndefinitely, long timeout, TimeUnit unit)
+ throws ExecutionException, InterruptedException, TimeoutException {
+ if (!isDone) {
+ synchronized (lock) {
+ if (waitIndefinitely) {
+ while (!isDone) {
+ lock.wait();
+ }
+ } else {
+ long now = System.nanoTime();
+ long deadline = now + unit.toNanos(timeout);
+ while (!isDone && deadline > now) {
+ lock.wait(TimeUnit.NANOSECONDS.toMillis(deadline - now));
+ now = System.nanoTime();
+ }
+ if (!isDone) {
+ throw new TimeoutException();
+ }
+ }
+ }
+ }
+ if (isCancelled) {
+ throw new CancellationException();
+ }
+ if (error != null) {
+ throw new ExecutionException(error);
+ }
+ return result;
+ }
+
+ private void setResult(T result, Throwable error, State newState) {
+ if (!isDone) {
+ synchronized (lock) {
+ if (!isDone) {
+ this.result = result;
+ this.error = error;
+ this.isDone = true;
+ changeState(newState);
+ }
+ lock.notifyAll();
+ }
+ }
+ }
+
+ private class JobPollTask implements Runnable {
+
+ private long currentInterval;
+
+ JobPollTask(long currentInterval) {
+ this.currentInterval = currentInterval;
+ }
+
+ @Override
+ public void run() {
+ try {
+ JobStatus status = conn.get(JobStatus.class, "/%d/jobs/%d", sessionId, jobId);
+ T result = null;
+ Throwable error = null;
+ boolean finished = false;
+
+ switch (status.state) {
+ case SUCCEEDED:
+ if (status.result != null) {
+ @SuppressWarnings("unchecked")
+ T localResult = (T) serializer.deserialize(ByteBuffer.wrap(status.result));
+ result = localResult;
+ }
+ finished = true;
+ break;
+
+ case FAILED:
+ // TODO: better exception.
+ error = new RuntimeException(status.error);
+ finished = true;
+ break;
+
+ case CANCELLED:
+ isCancelled = true;
+ finished = true;
+ break;
+
+ default:
+ // Nothing to do.
+ }
+ if (finished) {
+ setResult(result, error, status.state);
+ } else if (status.state != state) {
+ changeState(status.state);
+ }
+ if (!finished) {
+ currentInterval = Math.min(currentInterval * 2, maxPollInterval);
+ pollTask = executor.schedule(this, currentInterval, TimeUnit.MILLISECONDS);
+ }
+ } catch (Exception e) {
+ setResult(null, e, State.FAILED);
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/java/org/apache/livy/client/http/LivyConnection.java
----------------------------------------------------------------------
diff --git a/client-http/src/main/java/org/apache/livy/client/http/LivyConnection.java b/client-http/src/main/java/org/apache/livy/client/http/LivyConnection.java
new file mode 100644
index 0000000..e8612bf
--- /dev/null
+++ b/client-http/src/main/java/org/apache/livy/client/http/LivyConnection.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.http;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.security.Principal;
+import java.util.concurrent.TimeUnit;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.*;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
+import org.apache.http.entity.mime.content.FileBody;
+import org.apache.http.impl.NoConnectionReuseStrategy;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
+import org.apache.http.util.EntityUtils;
+
+import static org.apache.livy.client.http.HttpConf.Entry.*;
+
+/**
+ * Abstracts a connection to the Livy server; serializes multiple requests so that we only need
+ * one active HTTP connection (to keep resource usage down).
+ */
+class LivyConnection {
+
+ static final String SESSIONS_URI = "/sessions";
+ private static final String APPLICATION_JSON = "application/json";
+
+ private final URI server;
+ private final String uriRoot;
+ private final CloseableHttpClient client;
+ private final ObjectMapper mapper;
+
+ LivyConnection(URI uri, final HttpConf config) {
+ HttpClientContext ctx = HttpClientContext.create();
+ int port = uri.getPort() > 0 ? uri.getPort() : 8998;
+
+ String path = uri.getPath() != null ? uri.getPath() : "";
+ this.uriRoot = path + SESSIONS_URI;
+
+ RequestConfig reqConfig = new RequestConfig() {
+ @Override
+ public int getConnectTimeout() {
+ return (int) config.getTimeAsMs(CONNECTION_TIMEOUT);
+ }
+
+ @Override
+ public int getSocketTimeout() {
+ return (int) config.getTimeAsMs(SOCKET_TIMEOUT);
+ }
+
+ @Override
+ public boolean isAuthenticationEnabled() {
+ return true;
+ }
+
+ @Override
+ public boolean isContentCompressionEnabled() {
+ return config.getBoolean(CONTENT_COMPRESS_ENABLE);
+ }
+ };
+
+ Credentials credentials;
+ // If user info is specified in the url, pass them to the CredentialsProvider.
+ if (uri.getUserInfo() != null) {
+ String[] userInfo = uri.getUserInfo().split(":");
+ if (userInfo.length < 1) {
+ throw new IllegalArgumentException("Malformed user info in the url.");
+ }
+ try {
+ String username = URLDecoder.decode(userInfo[0], StandardCharsets.UTF_8.name());
+ String password = "";
+ if (userInfo.length > 1) {
+ password = URLDecoder.decode(userInfo[1], StandardCharsets.UTF_8.name());
+ }
+ credentials = new UsernamePasswordCredentials(username, password);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("User info in the url contains bad characters.", e);
+ }
+ } else {
+ credentials = new Credentials() {
+ @Override
+ public String getPassword() {
+ return null;
+ }
+
+ @Override
+ public Principal getUserPrincipal() {
+ return null;
+ }
+ };
+ }
+
+ CredentialsProvider credsProvider = new BasicCredentialsProvider();
+ credsProvider.setCredentials(AuthScope.ANY, credentials);
+
+ HttpClientBuilder builder = HttpClientBuilder.create()
+ .disableAutomaticRetries()
+ .evictExpiredConnections()
+ .evictIdleConnections(config.getTimeAsMs(CONNECTION_IDLE_TIMEOUT), TimeUnit.MILLISECONDS)
+ .setConnectionManager(new BasicHttpClientConnectionManager())
+ .setConnectionReuseStrategy(new NoConnectionReuseStrategy())
+ .setDefaultRequestConfig(reqConfig)
+ .setMaxConnTotal(1)
+ .setDefaultCredentialsProvider(credsProvider)
+ .setUserAgent("livy-client-http");
+
+ if (config.isSpnegoEnabled()) {
+ Registry<AuthSchemeProvider> authSchemeProviderRegistry =
+ RegistryBuilder.<AuthSchemeProvider>create()
+ .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
+ .build();
+ builder.setDefaultAuthSchemeRegistry(authSchemeProviderRegistry);
+ }
+
+ this.server = uri;
+ this.client = builder.build();
+ this.mapper = new ObjectMapper();
+ }
+
+ synchronized void close() throws IOException {
+ client.close();
+ }
+
+ synchronized <V> V delete(Class<V> retType, String uri, Object... uriParams) throws Exception {
+ return sendJSONRequest(new HttpDelete(), retType, uri, uriParams);
+ }
+
+ synchronized <V> V get(Class<V> retType, String uri, Object... uriParams) throws Exception {
+ return sendJSONRequest(new HttpGet(), retType, uri, uriParams);
+ }
+
+ synchronized <V> V post(
+ Object body,
+ Class<V> retType,
+ String uri,
+ Object... uriParams) throws Exception {
+ HttpPost post = new HttpPost();
+ if (body != null) {
+ byte[] bodyBytes = mapper.writeValueAsBytes(body);
+ post.setEntity(new ByteArrayEntity(bodyBytes));
+ }
+ return sendJSONRequest(post, retType, uri, uriParams);
+ }
+
+ synchronized <V> V post(
+ File f,
+ Class<V> retType,
+ String paramName,
+ String uri,
+ Object... uriParams) throws Exception {
+ HttpPost post = new HttpPost();
+ MultipartEntityBuilder builder = MultipartEntityBuilder.create();
+ builder.addPart(paramName, new FileBody(f));
+ post.setEntity(builder.build());
+ return sendRequest(post, retType, uri, uriParams);
+ }
+
+ private <V> V sendJSONRequest(
+ HttpRequestBase req,
+ Class<V> retType,
+ String uri,
+ Object... uriParams) throws Exception {
+ req.setHeader(HttpHeaders.ACCEPT, APPLICATION_JSON);
+ req.setHeader(HttpHeaders.CONTENT_TYPE, APPLICATION_JSON);
+ req.setHeader(HttpHeaders.CONTENT_ENCODING, "UTF-8");
+ return sendRequest(req, retType, uri, uriParams);
+ }
+
+ private <V> V sendRequest(
+ HttpRequestBase req,
+ Class<V> retType,
+ String uri,
+ Object... uriParams) throws Exception {
+ req.setURI(new URI(server.getScheme(), null, server.getHost(), server.getPort(),
+ uriRoot + String.format(uri, uriParams), null, null));
+ // It is no harm to set X-Requested-By when csrf protection is disabled.
+ if (req instanceof HttpPost || req instanceof HttpDelete || req instanceof HttpPut
+ || req instanceof HttpPatch) {
+ req.addHeader("X-Requested-By", "livy");
+ }
+ try (CloseableHttpResponse res = client.execute(req)) {
+ int status = (res.getStatusLine().getStatusCode() / 100) * 100;
+ HttpEntity entity = res.getEntity();
+ if (status == HttpStatus.SC_OK) {
+ if (!Void.class.equals(retType)) {
+ return mapper.readValue(entity.getContent(), retType);
+ } else {
+ return null;
+ }
+ } else {
+ String error = EntityUtils.toString(entity);
+ throw new IOException(String.format("%s: %s", res.getStatusLine().getReasonPhrase(),
+ error));
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/resources/META-INF/services/com.cloudera.livy.LivyClientFactory
----------------------------------------------------------------------
diff --git a/client-http/src/main/resources/META-INF/services/com.cloudera.livy.LivyClientFactory b/client-http/src/main/resources/META-INF/services/com.cloudera.livy.LivyClientFactory
deleted file mode 100644
index b186c05..0000000
--- a/client-http/src/main/resources/META-INF/services/com.cloudera.livy.LivyClientFactory
+++ /dev/null
@@ -1 +0,0 @@
-com.cloudera.livy.client.http.HttpClientFactory
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/main/resources/META-INF/services/org.apache.livy.LivyClientFactory
----------------------------------------------------------------------
diff --git a/client-http/src/main/resources/META-INF/services/org.apache.livy.LivyClientFactory b/client-http/src/main/resources/META-INF/services/org.apache.livy.LivyClientFactory
new file mode 100644
index 0000000..5dd53aa
--- /dev/null
+++ b/client-http/src/main/resources/META-INF/services/org.apache.livy.LivyClientFactory
@@ -0,0 +1 @@
+org.apache.livy.client.http.HttpClientFactory
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/test/scala/com/cloudera/livy/client/http/HttpClientSpec.scala
----------------------------------------------------------------------
diff --git a/client-http/src/test/scala/com/cloudera/livy/client/http/HttpClientSpec.scala b/client-http/src/test/scala/com/cloudera/livy/client/http/HttpClientSpec.scala
deleted file mode 100644
index f3ebcd5..0000000
--- a/client-http/src/test/scala/com/cloudera/livy/client/http/HttpClientSpec.scala
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.http
-
-import java.io.{File, InputStream}
-import java.net.{InetAddress, URI}
-import java.nio.file.{Files, Paths}
-import java.util.concurrent.{Future => JFuture, _}
-import java.util.concurrent.atomic.AtomicLong
-import javax.servlet.ServletContext
-import javax.servlet.http.HttpServletRequest
-
-import scala.concurrent.{ExecutionContext, Future}
-
-import org.mockito.ArgumentCaptor
-import org.mockito.Matchers.{eq => meq, _}
-import org.mockito.Mockito._
-import org.scalatest.{BeforeAndAfterAll, FunSpecLike}
-import org.scalatra.LifeCycle
-import org.scalatra.servlet.ScalatraListener
-
-import com.cloudera.livy._
-import com.cloudera.livy.client.common.{BufferUtils, Serializer}
-import com.cloudera.livy.client.common.HttpMessages._
-import com.cloudera.livy.server.WebServer
-import com.cloudera.livy.server.interactive.{InteractiveSession, InteractiveSessionServlet}
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.{InteractiveSessionManager, SessionState, Spark}
-import com.cloudera.livy.test.jobs.Echo
-import com.cloudera.livy.utils.AppInfo
-
-/**
- * The test for the HTTP client is written in Scala so we can reuse the code in the livy-server
- * module, which implements the client session backend. The client servlet has some functionality
- * overridden to avoid creating sub-processes for each seession.
- */
-class HttpClientSpec extends FunSpecLike with BeforeAndAfterAll with LivyBaseUnitTestSuite {
-
- import HttpClientSpec._
-
- private val TIMEOUT_S = 10
- private val ID_GENERATOR = new AtomicLong()
- private val serializer = new Serializer()
-
- private var server: WebServer = _
- private var client: LivyClient = _
-
- override def beforeAll(): Unit = {
- super.beforeAll()
- server = new WebServer(new LivyConf(), "0.0.0.0", 0)
-
- server.context.setResourceBase("src/main/com/cloudera/livy/server")
- server.context.setInitParameter(ScalatraListener.LifeCycleKey,
- classOf[HttpClientTestBootstrap].getCanonicalName)
- server.context.addEventListener(new ScalatraListener)
-
- server.start()
- }
-
- override def afterAll(): Unit = {
- super.afterAll()
- if (server != null) {
- server.stop()
- server = null
- }
- if (client != null) {
- client.stop(true)
- client = null
- }
- session = null
- }
-
- describe("HTTP client library") {
-
- it("should create clients") {
- // WebServer does this internally instead of respecting "0.0.0.0", so try to use the same
- // address.
- val uri = s"http://${InetAddress.getLocalHost.getHostAddress}:${server.port}/"
- client = new LivyClientBuilder(false).setURI(new URI(uri)).build()
- }
-
- withClient("should run and monitor asynchronous jobs") {
- testJob(false)
- }
-
- withClient("should propagate errors from jobs") {
- val errorMessage = "This job throws an error."
- val (jobId, handle) = runJob(false, { id => Seq(
- new JobStatus(id, JobHandle.State.FAILED, null, errorMessage))
- })
-
- val error = intercept[ExecutionException] {
- handle.get(TIMEOUT_S, TimeUnit.SECONDS)
- }
- assert(error.getCause() != null)
- assert(error.getCause().getMessage().indexOf(errorMessage) >= 0)
- verify(session, times(1)).jobStatus(meq(jobId))
- }
-
- withClient("should run and monitor synchronous jobs") {
- testJob(false)
- }
-
- withClient("should add files and jars") {
- val furi = new URI("hdfs:file")
- val juri = new URI("hdfs:jar")
-
- client.addFile(furi).get(TIMEOUT_S, TimeUnit.SECONDS)
- client.addJar(juri).get(TIMEOUT_S, TimeUnit.SECONDS)
-
- verify(session, times(1)).addFile(meq(furi))
- verify(session, times(1)).addJar(meq(juri))
- }
-
- withClient("should upload files and jars") {
- uploadAndVerify("file")
- uploadAndVerify("jar")
- }
-
- withClient("should cancel jobs") {
- val (jobId, handle) = runJob(false, { id => Seq(
- new JobStatus(id, JobHandle.State.STARTED, null, null),
- new JobStatus(id, JobHandle.State.CANCELLED, null, null))
- })
- handle.cancel(true)
-
- intercept[CancellationException] {
- handle.get(TIMEOUT_S, TimeUnit.SECONDS)
- }
-
- verify(session, times(1)).cancelJob(meq(jobId))
- }
-
- withClient("should notify listeners of job completion") {
- val (jobId, handle) = runJob(false, { id => Seq(
- new JobStatus(id, JobHandle.State.STARTED, null, null),
- new JobStatus(id, JobHandle.State.SUCCEEDED, serialize(id), null))
- })
-
- val listener = mock(classOf[JobHandle.Listener[Long]])
- handle.asInstanceOf[JobHandle[Long]].addListener(listener)
-
- assert(handle.get(TIMEOUT_S, TimeUnit.SECONDS) === jobId)
- verify(listener, times(1)).onJobSucceeded(any(), any())
- }
-
- withClient("should time out handle get() call") {
- // JobHandleImpl does exponential backoff checking the result of a job. Given an initial
- // wait of 100ms, 4 iterations should result in a wait of 800ms, so the handle should at that
- // point timeout a wait of 100ms.
- val (jobId, handle) = runJob(false, { id => Seq(
- new JobStatus(id, JobHandle.State.STARTED, null, null),
- new JobStatus(id, JobHandle.State.STARTED, null, null),
- new JobStatus(id, JobHandle.State.STARTED, null, null),
- new JobStatus(id, JobHandle.State.SUCCEEDED, serialize(id), null))
- })
-
- intercept[TimeoutException] {
- handle.get(100, TimeUnit.MILLISECONDS)
- }
-
- assert(handle.get(TIMEOUT_S, TimeUnit.SECONDS) === jobId)
- }
-
- withClient("should handle null responses") {
- testJob(false, response = Some(null))
- }
-
- withClient("should connect to existing sessions") {
- var sid = client.asInstanceOf[HttpClient].getSessionId()
- val uri = s"http://${InetAddress.getLocalHost.getHostAddress}:${server.port}" +
- s"${LivyConnection.SESSIONS_URI}/$sid"
- val newClient = new LivyClientBuilder(false).setURI(new URI(uri)).build()
- newClient.stop(false)
- verify(session, never()).stop()
- }
-
- withClient("should tear down clients") {
- client.stop(true)
- verify(session, times(1)).stop()
- }
-
- }
-
- private def uploadAndVerify(cmd: String): Unit = {
- val f = File.createTempFile("uploadTestFile", cmd)
- val expectedStr = "Test data"
- val expectedData = expectedStr.getBytes()
- Files.write(Paths.get(f.getAbsolutePath), expectedData)
- val b = new Array[Byte](expectedData.length)
- val captor = ArgumentCaptor.forClass(classOf[InputStream])
- if (cmd == "file") {
- client.uploadFile(f).get(TIMEOUT_S, TimeUnit.SECONDS)
- verify(session, times(1)).addFile(captor.capture(), meq(f.getName))
- } else {
- client.uploadJar(f).get(TIMEOUT_S, TimeUnit.SECONDS)
- verify(session, times(1)).addJar(captor.capture(), meq(f.getName))
- }
- captor.getValue.read(b)
- assert(expectedStr === new String(b))
- }
-
- private def runJob(sync: Boolean, genStatusFn: Long => Seq[JobStatus]): (Long, JFuture[Int]) = {
- val jobId = java.lang.Long.valueOf(ID_GENERATOR.incrementAndGet())
- when(session.submitJob(any(classOf[Array[Byte]]))).thenReturn(jobId)
-
- val statuses = genStatusFn(jobId)
- val first = statuses.head
- val remaining = statuses.drop(1)
- when(session.jobStatus(meq(jobId))).thenReturn(first, remaining: _*)
-
- val job = new Echo(42)
- val handle = if (sync) client.run(job) else client.submit(job)
- (jobId, handle)
- }
-
- private def testJob(sync: Boolean, response: Option[Any] = None): Unit = {
- val (jobId, handle) = runJob(sync, { id => Seq(
- new JobStatus(id, JobHandle.State.STARTED, null, null),
- new JobStatus(id, JobHandle.State.SUCCEEDED, serialize(response.getOrElse(id)), null))
- })
- assert(handle.get(TIMEOUT_S, TimeUnit.SECONDS) === response.getOrElse(jobId))
- verify(session, times(2)).jobStatus(meq(jobId))
- }
-
- private def withClient(desc: String)(fn: => Unit): Unit = {
- it(desc) {
- assume(client != null, "No active client.")
- fn
- }
- }
-
- def serialize(value: Any): Array[Byte] = {
- BufferUtils.toByteArray(serializer.serialize(value))
- }
-
-}
-
-private object HttpClientSpec {
-
- // Hack warning: keep the session object available so that individual tests can mock
- // the desired behavior before making requests to the server.
- var session: InteractiveSession = _
-
-}
-
-private class HttpClientTestBootstrap extends LifeCycle {
-
- private implicit def executor: ExecutionContext = ExecutionContext.global
-
- override def init(context: ServletContext): Unit = {
- val conf = new LivyConf()
- val stateStore = mock(classOf[SessionStore])
- val sessionManager = new InteractiveSessionManager(conf, stateStore, Some(Seq.empty))
- val servlet = new InteractiveSessionServlet(sessionManager, stateStore, conf) {
- override protected def createSession(req: HttpServletRequest): InteractiveSession = {
- val session = mock(classOf[InteractiveSession])
- val id = sessionManager.nextId()
- when(session.id).thenReturn(id)
- when(session.appId).thenReturn(None)
- when(session.appInfo).thenReturn(AppInfo())
- when(session.state).thenReturn(SessionState.Idle())
- when(session.proxyUser).thenReturn(None)
- when(session.kind).thenReturn(Spark())
- when(session.stop()).thenReturn(Future.successful(()))
- require(HttpClientSpec.session == null, "Session already created?")
- HttpClientSpec.session = session
- session
- }
- }
-
- context.mount(servlet, s"${LivyConnection.SESSIONS_URI}/*")
- }
-
-}
[24/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/src/test/scala/org/apache/livy/repl/SparkSessionSpec.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/livy/repl/SparkSessionSpec.scala b/repl/src/test/scala/org/apache/livy/repl/SparkSessionSpec.scala
new file mode 100644
index 0000000..9369519
--- /dev/null
+++ b/repl/src/test/scala/org/apache/livy/repl/SparkSessionSpec.scala
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.apache.spark.SparkConf
+import org.json4s.Extraction
+import org.json4s.JsonAST.JValue
+import org.json4s.jackson.JsonMethods.parse
+import org.scalatest.concurrent.Eventually._
+
+import org.apache.livy.rsc.RSCConf
+import org.apache.livy.rsc.driver.StatementState
+
+class SparkSessionSpec extends BaseSessionSpec {
+
+ override def createInterpreter(): Interpreter = new SparkInterpreter(new SparkConf())
+
+ it should "execute `1 + 2` == 3" in withSession { session =>
+ val statement = execute(session)("1 + 2")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "res0: Int = 3"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "execute `x = 1`, then `y = 2`, then `x + y`" in withSession { session =>
+ val executeWithSession = execute(session)(_)
+ var statement = executeWithSession("val x = 1")
+ statement.id should equal (0)
+
+ var result = parse(statement.output)
+ var expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "x: Int = 1"
+ )
+ ))
+
+ result should equal (expectedResult)
+
+ statement = executeWithSession("val y = 2")
+ statement.id should equal (1)
+
+ result = parse(statement.output)
+ expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 1,
+ "data" -> Map(
+ "text/plain" -> "y: Int = 2"
+ )
+ ))
+
+ result should equal (expectedResult)
+
+ statement = executeWithSession("x + y")
+ statement.id should equal (2)
+
+ result = parse(statement.output)
+ expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 2,
+ "data" -> Map(
+ "text/plain" -> "res0: Int = 3"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "capture stdout" in withSession { session =>
+ val statement = execute(session)("""println("Hello World")""")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "Hello World"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "report an error if accessing an unknown variable" in withSession { session =>
+ val statement = execute(session)("""x""")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+
+ def extract(key: String): String = (result \ key).extract[String]
+
+ extract("status") should equal ("error")
+ extract("execution_count") should equal ("0")
+ extract("ename") should equal ("Error")
+ extract("evalue") should include ("error: not found: value x")
+ }
+
+ it should "report an error if exception is thrown" in withSession { session =>
+ val statement = execute(session)(
+ """def func1() {
+ |throw new Exception()
+ |}
+ |func1()""".stripMargin)
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val resultMap = result.extract[Map[String, JValue]]
+
+ // Manually extract the values since the line numbers in the exception could change.
+ resultMap("status").extract[String] should equal ("error")
+ resultMap("execution_count").extract[Int] should equal (0)
+ resultMap("ename").extract[String] should equal ("Error")
+ resultMap("evalue").extract[String] should include ("java.lang.Exception")
+
+ val traceback = resultMap("traceback").extract[Seq[String]]
+ traceback(0) should include ("func1(<console>:")
+ }
+
+ it should "access the spark context" in withSession { session =>
+ val statement = execute(session)("""sc""")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+ val resultMap = result.extract[Map[String, JValue]]
+
+ // Manually extract the values since the line numbers in the exception could change.
+ resultMap("status").extract[String] should equal ("ok")
+ resultMap("execution_count").extract[Int] should equal (0)
+
+ val data = resultMap("data").extract[Map[String, JValue]]
+ data("text/plain").extract[String] should include (
+ "res0: org.apache.spark.SparkContext = org.apache.spark.SparkContext")
+ }
+
+ it should "execute spark commands" in withSession { session =>
+ val statement = execute(session)(
+ """sc.parallelize(0 to 1).map{i => i+1}.collect""".stripMargin)
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "text/plain" -> "res0: Array[Int] = Array(1, 2)"
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "do table magic" in withSession { session =>
+ val statement = execute(session)("val x = List((1, \"a\"), (3, \"b\"))\n%table x")
+ statement.id should equal (0)
+
+ val result = parse(statement.output)
+
+ val expectedResult = Extraction.decompose(Map(
+ "status" -> "ok",
+ "execution_count" -> 0,
+ "data" -> Map(
+ "application/vnd.livy.table.v1+json" -> Map(
+ "headers" -> List(
+ Map("type" -> "BIGINT_TYPE", "name" -> "_1"),
+ Map("type" -> "STRING_TYPE", "name" -> "_2")),
+ "data" -> List(List(1, "a"), List(3, "b"))
+ )
+ )
+ ))
+
+ result should equal (expectedResult)
+ }
+
+ it should "cancel spark jobs" in withSession { session =>
+ val stmtId = session.execute(
+ """sc.parallelize(0 to 10).map { i => Thread.sleep(10000); i + 1 }.collect""".stripMargin)
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ assert(session.statements(stmtId).state.get() == StatementState.Running)
+ }
+ session.cancel(stmtId)
+
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ assert(session.statements(stmtId).state.get() == StatementState.Cancelled)
+ session.statements(stmtId).output should include (
+ "Job 0 cancelled part of cancelled job group 0")
+ }
+ }
+
+ it should "cancel waiting statement" in withSession { session =>
+ val stmtId1 = session.execute(
+ """sc.parallelize(0 to 10).map { i => Thread.sleep(10000); i + 1 }.collect""".stripMargin)
+ val stmtId2 = session.execute(
+ """sc.parallelize(0 to 10).map { i => Thread.sleep(10000); i + 1 }.collect""".stripMargin)
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ assert(session.statements(stmtId1).state.get() == StatementState.Running)
+ }
+
+ assert(session.statements(stmtId2).state.get() == StatementState.Waiting)
+
+ session.cancel(stmtId2)
+ assert(session.statements(stmtId2).state.get() == StatementState.Cancelled)
+
+ session.cancel(stmtId1)
+ assert(session.statements(stmtId1).state.get() == StatementState.Cancelling)
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ assert(session.statements(stmtId1).state.get() == StatementState.Cancelled)
+ session.statements(stmtId1).output should include (
+ "Job 0 cancelled part of cancelled job group 0")
+ }
+ }
+
+ it should "correctly calculate progress" in withSession { session =>
+ val executeCode =
+ """
+ |sc.parallelize(1 to 2, 2).map(i => (i, 1)).collect()
+ """.stripMargin
+
+ val stmtId = session.execute(executeCode)
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ session.progressOfStatement(stmtId) should be(1.0)
+ }
+ }
+
+ it should "not generate Spark jobs for plain Scala code" in withSession { session =>
+ val executeCode = """1 + 1"""
+
+ val stmtId = session.execute(executeCode)
+ session.progressOfStatement(stmtId) should be (0.0)
+ }
+
+ it should "handle multiple jobs in one statement" in withSession { session =>
+ val executeCode =
+ """
+ |sc.parallelize(1 to 2, 2).map(i => (i, 1)).collect()
+ |sc.parallelize(1 to 2, 2).map(i => (i, 1)).collect()
+ """.stripMargin
+
+ val stmtId = session.execute(executeCode)
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ session.progressOfStatement(stmtId) should be(1.0)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/pom.xml
----------------------------------------------------------------------
diff --git a/rsc/pom.xml b/rsc/pom.xml
index 7ff8c2a..660ca35 100644
--- a/rsc/pom.xml
+++ b/rsc/pom.xml
@@ -19,14 +19,14 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
</parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-rsc</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -35,17 +35,17 @@
<dependencies>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-client-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-test-lib</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -133,7 +133,7 @@
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
- <include>com.cloudera.livy:livy-client-common</include>
+ <include>org.apache.livy:livy-client-common</include>
<include>com.esotericsoftware.kryo:kryo</include>
</includes>
</artifactSet>
@@ -150,7 +150,7 @@
<relocations>
<relocation>
<pattern>com.esotericsoftware</pattern>
- <shadedPattern>com.cloudera.livy.shaded.kryo</shadedPattern>
+ <shadedPattern>org.apache.livy.shaded.kryo</shadedPattern>
</relocation>
</relocations>
<outputFile>${project.build.directory}/jars/${project.artifactId}-${project.version}.jar</outputFile>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/BaseProtocol.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/BaseProtocol.java b/rsc/src/main/java/com/cloudera/livy/rsc/BaseProtocol.java
deleted file mode 100644
index 419c1f7..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/BaseProtocol.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.rsc.driver.Statement;
-import com.cloudera.livy.rsc.rpc.RpcDispatcher;
-
-public abstract class BaseProtocol extends RpcDispatcher {
-
- protected static class CancelJob {
-
- public final String id;
-
- CancelJob(String id) {
- this.id = id;
- }
-
- CancelJob() {
- this(null);
- }
-
- }
-
- protected static class EndSession {
-
- }
-
- protected static class Error {
-
- public final String cause;
-
- public Error(Throwable cause) {
- if (cause == null) {
- this.cause = "";
- } else {
- this.cause = Utils.stackTraceAsString(cause);
- }
- }
-
- public Error() {
- this(null);
- }
-
- }
-
- public static class BypassJobRequest {
-
- public final String id;
- public final byte[] serializedJob;
- public final boolean synchronous;
-
- public BypassJobRequest(String id, byte[] serializedJob, boolean synchronous) {
- this.id = id;
- this.serializedJob = serializedJob;
- this.synchronous = synchronous;
- }
-
- public BypassJobRequest() {
- this(null, null, false);
- }
-
- }
-
- protected static class GetBypassJobStatus {
-
- public final String id;
-
- public GetBypassJobStatus(String id) {
- this.id = id;
- }
-
- public GetBypassJobStatus() {
- this(null);
- }
-
- }
-
- protected static class JobRequest<T> {
-
- public final String id;
- public final Job<T> job;
-
- public JobRequest(String id, Job<T> job) {
- this.id = id;
- this.job = job;
- }
-
- public JobRequest() {
- this(null, null);
- }
-
- }
-
- protected static class JobResult<T> {
-
- public final String id;
- public final T result;
- public final String error;
-
- public JobResult(String id, T result, Throwable error) {
- this.id = id;
- this.result = result;
- this.error = error != null ? Utils.stackTraceAsString(error) : null;
- }
-
- public JobResult() {
- this(null, null, null);
- }
-
- }
-
- protected static class JobStarted {
-
- public final String id;
-
- public JobStarted(String id) {
- this.id = id;
- }
-
- public JobStarted() {
- this(null);
- }
-
- }
-
- protected static class SyncJobRequest<T> {
-
- public final Job<T> job;
-
- public SyncJobRequest(Job<T> job) {
- this.job = job;
- }
-
- public SyncJobRequest() {
- this(null);
- }
-
- }
-
- public static class RemoteDriverAddress {
-
- public final String host;
- public final int port;
-
- public RemoteDriverAddress(String host, int port) {
- this.host = host;
- this.port = port;
- }
-
- public RemoteDriverAddress() {
- this(null, -1);
- }
-
- }
-
- public static class ReplJobRequest {
-
- public final String code;
-
- public ReplJobRequest(String code) {
- this.code = code;
- }
-
- public ReplJobRequest() {
- this(null);
- }
- }
-
- public static class GetReplJobResults {
- public boolean allResults;
- public Integer from, size;
-
- public GetReplJobResults(Integer from, Integer size) {
- this.allResults = false;
- this.from = from;
- this.size = size;
- }
-
- public GetReplJobResults() {
- this.allResults = true;
- from = null;
- size = null;
- }
- }
-
- protected static class ReplState {
-
- public final String state;
-
- public ReplState(String state) {
- this.state = state;
- }
-
- public ReplState() {
- this(null);
- }
- }
-
- public static class CancelReplJobRequest {
- public final int id;
-
- public CancelReplJobRequest(int id) {
- this.id = id;
- }
-
- public CancelReplJobRequest() {
- this(-1);
- }
- }
-
- public static class InitializationError {
-
- public final String stackTrace;
-
- public InitializationError(String stackTrace) {
- this.stackTrace = stackTrace;
- }
-
- public InitializationError() {
- this(null);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/BypassJobStatus.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/BypassJobStatus.java b/rsc/src/main/java/com/cloudera/livy/rsc/BypassJobStatus.java
deleted file mode 100644
index 7f35cc7..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/BypassJobStatus.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import java.util.List;
-
-import com.cloudera.livy.JobHandle;
-
-public class BypassJobStatus {
-
- public final JobHandle.State state;
- public final byte[] result;
- public final String error;
-
- public BypassJobStatus(JobHandle.State state, byte[] result, String error) {
- this.state = state;
- this.result = result;
- this.error = error;
- }
-
- BypassJobStatus() {
- this(null, null, null);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/ContextInfo.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/ContextInfo.java b/rsc/src/main/java/com/cloudera/livy/rsc/ContextInfo.java
deleted file mode 100644
index 5e930cd..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/ContextInfo.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-/**
- * Information about a running RSC instance.
- */
-class ContextInfo {
-
- final String remoteAddress;
- final int remotePort;
- final String clientId;
- final String secret;
-
- ContextInfo(String remoteAddress, int remotePort, String clientId, String secret) {
- this.remoteAddress = remoteAddress;
- this.remotePort = remotePort;
- this.clientId = clientId;
- this.secret = secret;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/ContextLauncher.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/ContextLauncher.java b/rsc/src/main/java/com/cloudera/livy/rsc/ContextLauncher.java
deleted file mode 100644
index 9a5e447..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/ContextLauncher.java
+++ /dev/null
@@ -1,456 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.Reader;
-import java.io.Writer;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.nio.file.attribute.PosixFilePermission.*;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.util.concurrent.Promise;
-import org.apache.spark.launcher.SparkLauncher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.client.common.TestUtils;
-import com.cloudera.livy.rsc.driver.RSCDriverBootstrapper;
-import com.cloudera.livy.rsc.rpc.Rpc;
-import com.cloudera.livy.rsc.rpc.RpcDispatcher;
-import com.cloudera.livy.rsc.rpc.RpcServer;
-import static com.cloudera.livy.rsc.RSCConf.Entry.*;
-
-/**
- * Encapsulates code needed to launch a new Spark context and collect information about how
- * to establish a client connection to it.
- */
-class ContextLauncher {
-
- private static final Logger LOG = LoggerFactory.getLogger(ContextLauncher.class);
- private static final AtomicInteger CHILD_IDS = new AtomicInteger();
-
- private static final String SPARK_DEPLOY_MODE = "spark.submit.deployMode";
- private static final String SPARK_JARS_KEY = "spark.jars";
- private static final String SPARK_ARCHIVES_KEY = "spark.yarn.dist.archives";
- private static final String SPARK_HOME_ENV = "SPARK_HOME";
-
- static DriverProcessInfo create(RSCClientFactory factory, RSCConf conf)
- throws IOException {
- ContextLauncher launcher = new ContextLauncher(factory, conf);
- return new DriverProcessInfo(launcher.promise, launcher.child.child);
- }
-
- private final Promise<ContextInfo> promise;
- private final ScheduledFuture<?> timeout;
- private final String clientId;
- private final String secret;
- private final ChildProcess child;
- private final RSCConf conf;
- private final RSCClientFactory factory;
-
- private ContextLauncher(RSCClientFactory factory, RSCConf conf) throws IOException {
- this.promise = factory.getServer().getEventLoopGroup().next().newPromise();
- this.clientId = UUID.randomUUID().toString();
- this.secret = factory.getServer().createSecret();
- this.conf = conf;
- this.factory = factory;
-
- final RegistrationHandler handler = new RegistrationHandler();
- try {
- factory.getServer().registerClient(clientId, secret, handler);
- String replMode = conf.get("repl");
- boolean repl = replMode != null && replMode.equals("true");
-
- conf.set(LAUNCHER_ADDRESS, factory.getServer().getAddress());
- conf.set(LAUNCHER_PORT, factory.getServer().getPort());
- conf.set(CLIENT_ID, clientId);
- conf.set(CLIENT_SECRET, secret);
-
- Utils.addListener(promise, new FutureListener<ContextInfo>() {
- @Override
- public void onFailure(Throwable error) throws Exception {
- // If promise is cancelled or failed, make sure spark-submit is not leaked.
- if (child != null) {
- child.kill();
- }
- }
- });
-
- this.child = startDriver(conf, promise);
-
- // Set up a timeout to fail the promise if we don't hear back from the context
- // after a configurable timeout.
- Runnable timeoutTask = new Runnable() {
- @Override
- public void run() {
- connectTimeout(handler);
- }
- };
- this.timeout = factory.getServer().getEventLoopGroup().schedule(timeoutTask,
- conf.getTimeAsMs(RPC_CLIENT_HANDSHAKE_TIMEOUT), TimeUnit.MILLISECONDS);
- } catch (Exception e) {
- dispose(true);
- throw Utils.propagate(e);
- }
- }
-
- private void connectTimeout(RegistrationHandler handler) {
- if (promise.tryFailure(new TimeoutException("Timed out waiting for context to start."))) {
- handler.dispose();
- }
- dispose(true);
- }
-
- private void dispose(boolean forceKill) {
- factory.getServer().unregisterClient(clientId);
- try {
- if (child != null) {
- if (forceKill) {
- child.kill();
- } else {
- child.detach();
- }
- }
- } finally {
- factory.unref();
- }
- }
-
- private static ChildProcess startDriver(final RSCConf conf, Promise<?> promise)
- throws IOException {
- String livyJars = conf.get(LIVY_JARS);
- if (livyJars == null) {
- String livyHome = System.getenv("LIVY_HOME");
- Utils.checkState(livyHome != null,
- "Need one of LIVY_HOME or %s set.", LIVY_JARS.key());
- File rscJars = new File(livyHome, "rsc-jars");
- if (!rscJars.isDirectory()) {
- rscJars = new File(livyHome, "rsc/target/jars");
- }
- Utils.checkState(rscJars.isDirectory(),
- "Cannot find 'client-jars' directory under LIVY_HOME.");
- List<String> jars = new ArrayList<>();
- for (File f : rscJars.listFiles()) {
- jars.add(f.getAbsolutePath());
- }
- livyJars = Utils.join(jars, ",");
- }
- merge(conf, SPARK_JARS_KEY, livyJars, ",");
-
- String kind = conf.get(SESSION_KIND);
- if ("sparkr".equals(kind)) {
- merge(conf, SPARK_ARCHIVES_KEY, conf.get(RSCConf.Entry.SPARKR_PACKAGE), ",");
- } else if ("pyspark".equals(kind)) {
- merge(conf, "spark.submit.pyFiles", conf.get(RSCConf.Entry.PYSPARK_ARCHIVES), ",");
- }
-
- // Disable multiple attempts since the RPC server doesn't yet support multiple
- // connections for the same registered app.
- conf.set("spark.yarn.maxAppAttempts", "1");
-
- // Let the launcher go away when launcher in yarn cluster mode. This avoids keeping lots
- // of "small" Java processes lingering on the Livy server node.
- conf.set("spark.yarn.submit.waitAppCompletion", "false");
-
- if (!conf.getBoolean(CLIENT_IN_PROCESS) &&
- // For tests which doesn't shutdown RscDriver gracefully, JaCoCo exec isn't dumped properly.
- // Disable JaCoCo for this case.
- !conf.getBoolean(TEST_STUCK_END_SESSION)) {
- // For testing; propagate jacoco settings so that we also do coverage analysis
- // on the launched driver. We replace the name of the main file ("main.exec")
- // so that we don't end up fighting with the main test launcher.
- String jacocoArgs = TestUtils.getJacocoArgs();
- if (jacocoArgs != null) {
- merge(conf, SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, jacocoArgs, " ");
- }
- }
-
- final File confFile = writeConfToFile(conf);
-
- if (ContextLauncher.mockSparkSubmit != null) {
- LOG.warn("!!!! Using mock spark-submit. !!!!");
- return new ChildProcess(conf, promise, ContextLauncher.mockSparkSubmit, confFile);
- } else if (conf.getBoolean(CLIENT_IN_PROCESS)) {
- // Mostly for testing things quickly. Do not do this in production.
- LOG.warn("!!!! Running remote driver in-process. !!!!");
- Runnable child = new Runnable() {
- @Override
- public void run() {
- try {
- RSCDriverBootstrapper.main(new String[] { confFile.getAbsolutePath() });
- } catch (Exception e) {
- throw Utils.propagate(e);
- }
- }
- };
- return new ChildProcess(conf, promise, child, confFile);
- } else {
- final SparkLauncher launcher = new SparkLauncher();
-
- // Spark 1.x does not support specifying deploy mode in conf and needs special handling.
- String deployMode = conf.get(SPARK_DEPLOY_MODE);
- if (deployMode != null) {
- launcher.setDeployMode(deployMode);
- }
-
- launcher.setSparkHome(System.getenv(SPARK_HOME_ENV));
- launcher.setAppResource("spark-internal");
- launcher.setPropertiesFile(confFile.getAbsolutePath());
- launcher.setMainClass(RSCDriverBootstrapper.class.getName());
-
- if (conf.get(PROXY_USER) != null) {
- launcher.addSparkArg("--proxy-user", conf.get(PROXY_USER));
- }
-
- return new ChildProcess(conf, promise, launcher.launch(), confFile);
- }
- }
-
- private static void merge(RSCConf conf, String key, String livyConf, String sep) {
- String confValue = Utils.join(Arrays.asList(livyConf, conf.get(key)), sep);
- conf.set(key, confValue);
- }
-
- /**
- * Write the configuration to a file readable only by the process's owner. Livy properties
- * are written with an added prefix so that they can be loaded using SparkConf on the driver
- * side.
- *
- * The default Spark configuration (from either SPARK_HOME or SPARK_CONF_DIR) is merged into
- * the user configuration, so that defaults set by Livy's admin take effect when not overridden
- * by the user.
- */
- private static File writeConfToFile(RSCConf conf) throws IOException {
- Properties confView = new Properties();
- for (Map.Entry<String, String> e : conf) {
- String key = e.getKey();
- if (!key.startsWith(RSCConf.SPARK_CONF_PREFIX)) {
- key = RSCConf.LIVY_SPARK_PREFIX + key;
- }
- confView.setProperty(key, e.getValue());
- }
-
- // Load the default Spark configuration.
- String confDir = System.getenv("SPARK_CONF_DIR");
- if (confDir == null && System.getenv(SPARK_HOME_ENV) != null) {
- confDir = System.getenv(SPARK_HOME_ENV) + File.separator + "conf";
- }
-
- if (confDir != null) {
- File sparkDefaults = new File(confDir + File.separator + "spark-defaults.conf");
- if (sparkDefaults.isFile()) {
- Properties sparkConf = new Properties();
- Reader r = new InputStreamReader(new FileInputStream(sparkDefaults), UTF_8);
- try {
- sparkConf.load(r);
- } finally {
- r.close();
- }
-
- for (String key : sparkConf.stringPropertyNames()) {
- if (!confView.containsKey(key)) {
- confView.put(key, sparkConf.getProperty(key));
- }
- }
- }
- }
-
- File file = File.createTempFile("livyConf", ".properties");
- Files.setPosixFilePermissions(file.toPath(), EnumSet.of(OWNER_READ, OWNER_WRITE));
- //file.deleteOnExit();
-
- Writer writer = new OutputStreamWriter(new FileOutputStream(file), UTF_8);
- try {
- confView.store(writer, "Livy App Context Configuration");
- } finally {
- writer.close();
- }
-
- return file;
- }
-
-
- private class RegistrationHandler extends BaseProtocol
- implements RpcServer.ClientCallback {
-
- volatile RemoteDriverAddress driverAddress;
-
- private Rpc client;
-
- @Override
- public RpcDispatcher onNewClient(Rpc client) {
- LOG.debug("New RPC client connected from {}.", client.getChannel());
- this.client = client;
- return this;
- }
-
- @Override
- public void onSaslComplete(Rpc client) {
- }
-
- void dispose() {
- if (client != null) {
- client.close();
- }
- }
-
- private void handle(ChannelHandlerContext ctx, RemoteDriverAddress msg) {
- ContextInfo info = new ContextInfo(msg.host, msg.port, clientId, secret);
- if (promise.trySuccess(info)) {
- timeout.cancel(true);
- LOG.debug("Received driver info for client {}: {}/{}.", client.getChannel(),
- msg.host, msg.port);
- } else {
- LOG.warn("Connection established but promise is already finalized.");
- }
-
- ctx.executor().submit(new Runnable() {
- @Override
- public void run() {
- dispose();
- ContextLauncher.this.dispose(false);
- }
- });
- }
-
- }
-
- private static class ChildProcess {
-
- private final RSCConf conf;
- private final Promise<?> promise;
- private final Process child;
- private final Thread monitor;
- private final File confFile;
-
- public ChildProcess(RSCConf conf, Promise<?> promise, Runnable child, File confFile) {
- this.conf = conf;
- this.promise = promise;
- this.monitor = monitor(child, CHILD_IDS.incrementAndGet());
- this.child = null;
- this.confFile = confFile;
- }
-
- public ChildProcess(RSCConf conf, Promise<?> promise, final Process childProc, File confFile) {
- int childId = CHILD_IDS.incrementAndGet();
- this.conf = conf;
- this.promise = promise;
- this.child = childProc;
- this.confFile = confFile;
-
- Runnable monitorTask = new Runnable() {
- @Override
- public void run() {
- try {
- int exitCode = child.waitFor();
- if (exitCode != 0) {
- LOG.warn("Child process exited with code {}.", exitCode);
- fail(new IOException(String.format("Child process exited with code %d.", exitCode)));
- }
- } catch (InterruptedException ie) {
- LOG.warn("Waiting thread interrupted, killing child process.");
- Thread.interrupted();
- child.destroy();
- } catch (Exception e) {
- LOG.warn("Exception while waiting for child process.", e);
- }
- }
- };
- this.monitor = monitor(monitorTask, childId);
- }
-
- private void fail(Throwable error) {
- promise.tryFailure(error);
- }
-
- public void kill() {
- if (child != null) {
- child.destroy();
- }
- monitor.interrupt();
- detach();
-
- if (!monitor.isAlive()) {
- return;
- }
-
- // Last ditch effort.
- if (monitor.isAlive()) {
- LOG.warn("Timed out shutting down remote driver, interrupting...");
- monitor.interrupt();
- }
- }
-
- public void detach() {
- try {
- monitor.join(conf.getTimeAsMs(CLIENT_SHUTDOWN_TIMEOUT));
- } catch (InterruptedException ie) {
- LOG.debug("Interrupted before driver thread was finished.");
- }
- }
-
- private Thread monitor(final Runnable task, int childId) {
- Runnable wrappedTask = new Runnable() {
- @Override
- public void run() {
- try {
- task.run();
- } finally {
- confFile.delete();
- }
- }
- };
- Thread thread = new Thread(wrappedTask);
- thread.setDaemon(true);
- thread.setName("ContextLauncher-" + childId);
- thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(Thread t, Throwable e) {
- LOG.warn("Child task threw exception.", e);
- fail(e);
- }
- });
- thread.start();
- return thread;
- }
- }
-
- // Just for testing.
- static Process mockSparkSubmit;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/DriverProcessInfo.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/DriverProcessInfo.java b/rsc/src/main/java/com/cloudera/livy/rsc/DriverProcessInfo.java
deleted file mode 100644
index a224fd6..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/DriverProcessInfo.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import io.netty.util.concurrent.Promise;
-
-/**
- * Information about driver process and @{@link ContextInfo}
- */
-public class DriverProcessInfo {
-
- private Promise<ContextInfo> contextInfo;
- private transient Process driverProcess;
-
- public DriverProcessInfo(Promise<ContextInfo> contextInfo, Process driverProcess) {
- this.contextInfo = contextInfo;
- this.driverProcess = driverProcess;
- }
-
- public Promise<ContextInfo> getContextInfo() {
- return contextInfo;
- }
-
- public Process getDriverProcess() {
- return driverProcess;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/FutureListener.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/FutureListener.java b/rsc/src/main/java/com/cloudera/livy/rsc/FutureListener.java
deleted file mode 100644
index 2109c81..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/FutureListener.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-/** A simplified future listener for netty futures. See Utils.addListener(). */
-public abstract class FutureListener<T> {
-
- public void onSuccess(T result) throws Exception { }
-
- public void onFailure(Throwable error) throws Exception { }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/JobHandleImpl.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/JobHandleImpl.java b/rsc/src/main/java/com/cloudera/livy/rsc/JobHandleImpl.java
deleted file mode 100644
index 9146425..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/JobHandleImpl.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import io.netty.util.concurrent.Promise;
-
-import com.cloudera.livy.JobHandle;
-import com.cloudera.livy.client.common.AbstractJobHandle;
-
-/**
- * A handle to a submitted job. Allows for monitoring and controlling of the running remote job.
- */
-class JobHandleImpl<T> extends AbstractJobHandle<T> {
-
- private final RSCClient client;
- private final String jobId;
- private final Promise<T> promise;
- private volatile State state;
-
- JobHandleImpl(RSCClient client, Promise<T> promise, String jobId) {
- this.client = client;
- this.jobId = jobId;
- this.promise = promise;
- }
-
- /** Requests a running job to be cancelled. */
- @Override
- public boolean cancel(boolean mayInterrupt) {
- if (changeState(State.CANCELLED)) {
- client.cancel(jobId);
- promise.cancel(mayInterrupt);
- return true;
- }
- return false;
- }
-
- @Override
- public T get() throws ExecutionException, InterruptedException {
- return promise.get();
- }
-
- @Override
- public T get(long timeout, TimeUnit unit)
- throws ExecutionException, InterruptedException, TimeoutException {
- return promise.get(timeout, unit);
- }
-
- @Override
- public boolean isCancelled() {
- return promise.isCancelled();
- }
-
- @Override
- public boolean isDone() {
- return promise.isDone();
- }
-
- @Override
- protected T result() {
- return promise.getNow();
- }
-
- @Override
- protected Throwable error() {
- return promise.cause();
- }
-
- @SuppressWarnings("unchecked")
- void setSuccess(Object result) {
- // The synchronization here is not necessary, but tests depend on it.
- synchronized (listeners) {
- promise.setSuccess((T) result);
- changeState(State.SUCCEEDED);
- }
- }
-
- void setFailure(Throwable error) {
- // The synchronization here is not necessary, but tests depend on it.
- synchronized (listeners) {
- promise.setFailure(error);
- changeState(State.FAILED);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/PingJob.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/PingJob.java b/rsc/src/main/java/com/cloudera/livy/rsc/PingJob.java
deleted file mode 100644
index 088763a..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/PingJob.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-/** A job that can be used to check for liveness of the remote context. */
-public class PingJob implements Job<Void> {
-
- @Override
- public Void call(JobContext jc) {
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/RSCClient.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/RSCClient.java b/rsc/src/main/java/com/cloudera/livy/rsc/RSCClient.java
deleted file mode 100644
index 11cb0f6..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/RSCClient.java
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.util.concurrent.GenericFutureListener;
-import io.netty.util.concurrent.ImmediateEventExecutor;
-import io.netty.util.concurrent.Promise;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobHandle;
-import com.cloudera.livy.LivyClient;
-import com.cloudera.livy.client.common.BufferUtils;
-import com.cloudera.livy.rsc.driver.AddFileJob;
-import com.cloudera.livy.rsc.driver.AddJarJob;
-import com.cloudera.livy.rsc.rpc.Rpc;
-import static com.cloudera.livy.rsc.RSCConf.Entry.*;
-
-public class RSCClient implements LivyClient {
- private static final Logger LOG = LoggerFactory.getLogger(RSCClient.class);
- private static final AtomicInteger EXECUTOR_GROUP_ID = new AtomicInteger();
-
- private final RSCConf conf;
- private final Promise<ContextInfo> contextInfoPromise;
- private final Map<String, JobHandleImpl<?>> jobs;
- private final ClientProtocol protocol;
- private final Promise<Rpc> driverRpc;
- private final int executorGroupId;
- private final EventLoopGroup eventLoopGroup;
- private final Promise<URI> serverUriPromise;
-
- private ContextInfo contextInfo;
- private Process driverProcess;
- private volatile boolean isAlive;
- private volatile String replState;
-
- RSCClient(RSCConf conf, Promise<ContextInfo> ctx, Process driverProcess) throws IOException {
- this.conf = conf;
- this.contextInfoPromise = ctx;
- this.driverProcess = driverProcess;
- this.jobs = new ConcurrentHashMap<>();
- this.protocol = new ClientProtocol();
- this.driverRpc = ImmediateEventExecutor.INSTANCE.newPromise();
- this.executorGroupId = EXECUTOR_GROUP_ID.incrementAndGet();
- this.eventLoopGroup = new NioEventLoopGroup(
- conf.getInt(RPC_MAX_THREADS),
- Utils.newDaemonThreadFactory("RSCClient-" + executorGroupId + "-%d"));
- this.serverUriPromise = ImmediateEventExecutor.INSTANCE.newPromise();
-
- Utils.addListener(this.contextInfoPromise, new FutureListener<ContextInfo>() {
- @Override
- public void onSuccess(ContextInfo info) throws Exception {
- connectToContext(info);
- String url = String.format("rsc://%s:%s@%s:%d",
- info.clientId, info.secret, info.remoteAddress, info.remotePort);
- serverUriPromise.setSuccess(URI.create(url));
- }
-
- @Override
- public void onFailure(Throwable error) {
- connectionError(error);
- serverUriPromise.setFailure(error);
- }
- });
-
- isAlive = true;
- }
-
- public boolean isAlive() {
- return isAlive;
- }
-
- public Process getDriverProcess() {
- return driverProcess;
- }
-
- private synchronized void connectToContext(final ContextInfo info) throws Exception {
- this.contextInfo = info;
-
- try {
- Promise<Rpc> promise = Rpc.createClient(conf,
- eventLoopGroup,
- info.remoteAddress,
- info.remotePort,
- info.clientId,
- info.secret,
- protocol);
- Utils.addListener(promise, new FutureListener<Rpc>() {
- @Override
- public void onSuccess(Rpc rpc) throws Exception {
- driverRpc.setSuccess(rpc);
- Utils.addListener(rpc.getChannel().closeFuture(), new FutureListener<Void>() {
- @Override
- public void onSuccess(Void unused) {
- if (isAlive) {
- LOG.warn("Client RPC channel closed unexpectedly.");
- try {
- stop(false);
- } catch (Exception e) { /* stop() itself prints warning. */ }
- }
- }
- });
- LOG.debug("Connected to context {} ({}, {}).", info.clientId,
- rpc.getChannel(), executorGroupId);
- }
-
- @Override
- public void onFailure(Throwable error) throws Exception {
- driverRpc.setFailure(error);
- connectionError(error);
- }
- });
- } catch (Exception e) {
- connectionError(e);
- }
- }
-
- private void connectionError(Throwable error) {
- LOG.error("Failed to connect to context.", error);
- try {
- stop(false);
- } catch (Exception e) { /* stop() itself prints warning. */ }
- }
-
- private <T> io.netty.util.concurrent.Future<T> deferredCall(final Object msg,
- final Class<T> retType) {
- if (driverRpc.isSuccess()) {
- try {
- return driverRpc.get().call(msg, retType);
- } catch (Exception ie) {
- throw Utils.propagate(ie);
- }
- }
-
- // No driver RPC yet, so install a listener and return a promise that will be ready when
- // the driver is up and the message is actually delivered.
- final Promise<T> promise = eventLoopGroup.next().newPromise();
- final FutureListener<T> callListener = new FutureListener<T>() {
- @Override
- public void onSuccess(T value) throws Exception {
- promise.setSuccess(value);
- }
-
- @Override
- public void onFailure(Throwable error) throws Exception {
- promise.setFailure(error);
- }
- };
-
- Utils.addListener(driverRpc, new FutureListener<Rpc>() {
- @Override
- public void onSuccess(Rpc rpc) throws Exception {
- Utils.addListener(rpc.call(msg, retType), callListener);
- }
-
- @Override
- public void onFailure(Throwable error) throws Exception {
- promise.setFailure(error);
- }
- });
- return promise;
- }
-
- public Future<URI> getServerUri() {
- return serverUriPromise;
- }
-
- @Override
- public <T> JobHandle<T> submit(Job<T> job) {
- return protocol.submit(job);
- }
-
- @Override
- public <T> Future<T> run(Job<T> job) {
- return protocol.run(job);
- }
-
- @Override
- public synchronized void stop(boolean shutdownContext) {
- if (isAlive) {
- isAlive = false;
- try {
- this.contextInfoPromise.cancel(true);
-
- if (shutdownContext && driverRpc.isSuccess()) {
- protocol.endSession();
-
- // Because the remote context won't really reply to the end session message -
- // since it closes the channel while handling it, we wait for the RPC's channel
- // to close instead.
- long stopTimeout = conf.getTimeAsMs(CLIENT_SHUTDOWN_TIMEOUT);
- driverRpc.get().getChannel().closeFuture().get(stopTimeout,
- TimeUnit.MILLISECONDS);
- }
- } catch (Exception e) {
- LOG.warn("Exception while waiting for end session reply.", e);
- Utils.propagate(e);
- } finally {
- if (driverRpc.isSuccess()) {
- try {
- driverRpc.get().close();
- } catch (Exception e) {
- LOG.warn("Error stopping RPC.", e);
- }
- }
-
- // Report failure for all pending jobs, so that clients can react.
- for (Map.Entry<String, JobHandleImpl<?>> e : jobs.entrySet()) {
- LOG.info("Failing pending job {} due to shutdown.", e.getKey());
- e.getValue().setFailure(new IOException("RSCClient instance stopped."));
- }
-
- eventLoopGroup.shutdownGracefully();
- }
- if (contextInfo != null) {
- LOG.debug("Disconnected from context {}, shutdown = {}.", contextInfo.clientId,
- shutdownContext);
- }
- }
- }
-
- @Override
- public Future<?> uploadJar(File jar) {
- throw new UnsupportedOperationException("Use addJar to add the jar to the remote context!");
- }
-
- @Override
- public Future<?> addJar(URI uri) {
- return submit(new AddJarJob(uri.toString()));
- }
-
- @Override
- public Future<?> uploadFile(File file) {
- throw new UnsupportedOperationException("Use addFile to add the file to the remote context!");
- }
-
- @Override
- public Future<?> addFile(URI uri) {
- return submit(new AddFileJob(uri.toString()));
- }
-
- public String bypass(ByteBuffer serializedJob, boolean sync) {
- return protocol.bypass(serializedJob, sync);
- }
-
- public Future<BypassJobStatus> getBypassJobStatus(String id) {
- return protocol.getBypassJobStatus(id);
- }
-
- public void cancel(String jobId) {
- protocol.cancel(jobId);
- }
-
- ContextInfo getContextInfo() {
- return contextInfo;
- }
-
- public Future<Integer> submitReplCode(String code) throws Exception {
- return deferredCall(new BaseProtocol.ReplJobRequest(code), Integer.class);
- }
-
- public void cancelReplCode(int statementId) throws Exception {
- deferredCall(new BaseProtocol.CancelReplJobRequest(statementId), Void.class);
- }
-
- public Future<ReplJobResults> getReplJobResults(Integer from, Integer size) throws Exception {
- return deferredCall(new BaseProtocol.GetReplJobResults(from, size), ReplJobResults.class);
- }
-
- public Future<ReplJobResults> getReplJobResults() throws Exception {
- return deferredCall(new BaseProtocol.GetReplJobResults(), ReplJobResults.class);
- }
-
- /**
- * @return Return the repl state. If this's not connected to a repl session, it will return null.
- */
- public String getReplState() {
- return replState;
- }
-
- private class ClientProtocol extends BaseProtocol {
-
- <T> JobHandleImpl<T> submit(Job<T> job) {
- final String jobId = UUID.randomUUID().toString();
- Object msg = new JobRequest<T>(jobId, job);
-
- final Promise<T> promise = eventLoopGroup.next().newPromise();
- final JobHandleImpl<T> handle = new JobHandleImpl<T>(RSCClient.this,
- promise, jobId);
- jobs.put(jobId, handle);
-
- final io.netty.util.concurrent.Future<Void> rpc = deferredCall(msg, Void.class);
- LOG.debug("Sending JobRequest[{}].", jobId);
-
- Utils.addListener(rpc, new FutureListener<Void>() {
- @Override
- public void onSuccess(Void unused) throws Exception {
- handle.changeState(JobHandle.State.QUEUED);
- }
-
- @Override
- public void onFailure(Throwable error) throws Exception {
- error.printStackTrace();
- promise.tryFailure(error);
- }
- });
- promise.addListener(new GenericFutureListener<Promise<T>>() {
- @Override
- public void operationComplete(Promise<T> p) {
- if (jobId != null) {
- jobs.remove(jobId);
- }
- if (p.isCancelled() && !rpc.isDone()) {
- rpc.cancel(true);
- }
- }
- });
- return handle;
- }
-
- @SuppressWarnings("unchecked")
- <T> Future<T> run(Job<T> job) {
- return (Future<T>) deferredCall(new SyncJobRequest(job), Object.class);
- }
-
- String bypass(ByteBuffer serializedJob, boolean sync) {
- String jobId = UUID.randomUUID().toString();
- Object msg = new BypassJobRequest(jobId, BufferUtils.toByteArray(serializedJob), sync);
- deferredCall(msg, Void.class);
- return jobId;
- }
-
- Future<BypassJobStatus> getBypassJobStatus(String id) {
- return deferredCall(new GetBypassJobStatus(id), BypassJobStatus.class);
- }
-
- void cancel(String jobId) {
- deferredCall(new CancelJob(jobId), Void.class);
- }
-
- Future<?> endSession() {
- return deferredCall(new EndSession(), Void.class);
- }
-
- private void handle(ChannelHandlerContext ctx, InitializationError msg) {
- LOG.warn("Error reported from remote driver: %s", msg.stackTrace);
- }
-
- private void handle(ChannelHandlerContext ctx, JobResult msg) {
- JobHandleImpl<?> handle = jobs.remove(msg.id);
- if (handle != null) {
- LOG.info("Received result for {}", msg.id);
- // TODO: need a better exception for this.
- Throwable error = msg.error != null ? new RuntimeException(msg.error) : null;
- if (error == null) {
- handle.setSuccess(msg.result);
- } else {
- handle.setFailure(error);
- }
- } else {
- LOG.warn("Received result for unknown job {}", msg.id);
- }
- }
-
- private void handle(ChannelHandlerContext ctx, JobStarted msg) {
- JobHandleImpl<?> handle = jobs.get(msg.id);
- if (handle != null) {
- handle.changeState(JobHandle.State.STARTED);
- } else {
- LOG.warn("Received event for unknown job {}", msg.id);
- }
- }
-
- private void handle(ChannelHandlerContext ctx, ReplState msg) {
- LOG.trace("Received repl state for {}", msg.state);
- replState = msg.state;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/RSCClientFactory.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/RSCClientFactory.java b/rsc/src/main/java/com/cloudera/livy/rsc/RSCClientFactory.java
deleted file mode 100644
index 27540bb..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/RSCClientFactory.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import io.netty.util.concurrent.ImmediateEventExecutor;
-import io.netty.util.concurrent.Promise;
-
-import com.cloudera.livy.LivyClient;
-import com.cloudera.livy.LivyClientFactory;
-import com.cloudera.livy.rsc.rpc.RpcServer;
-
-/**
- * Factory for RSC clients.
- */
-public final class RSCClientFactory implements LivyClientFactory {
-
- private final AtomicInteger refCount = new AtomicInteger();
- private RpcServer server = null;
-
- /**
- * Creates a local Livy client if the URI has the "rsc" scheme.
- * <p>
- * If the URI contains user information, host and port, the library will try to connect to an
- * existing RSC instance with the provided information, and most of the provided configuration
- * will be ignored.
- * <p>
- * Otherwise, a new Spark context will be started with the given configuration.
- */
- @Override
- public LivyClient createClient(URI uri, Properties config) {
- if (!"rsc".equals(uri.getScheme())) {
- return null;
- }
-
- RSCConf lconf = new RSCConf(config);
-
- boolean needsServer = false;
- try {
- Promise<ContextInfo> info;
- Process driverProcess = null;
- if (uri.getUserInfo() != null && uri.getHost() != null && uri.getPort() > 0) {
- info = createContextInfo(uri);
- } else {
- needsServer = true;
- ref(lconf);
- DriverProcessInfo processInfo = ContextLauncher.create(this, lconf);
- info = processInfo.getContextInfo();
- driverProcess = processInfo.getDriverProcess();
- }
- return new RSCClient(lconf, info, driverProcess);
- } catch (Exception e) {
- if (needsServer) {
- unref();
- }
- throw Utils.propagate(e);
- }
- }
-
- RpcServer getServer() {
- return server;
- }
-
- private synchronized void ref(RSCConf config) throws IOException {
- if (refCount.get() != 0) {
- refCount.incrementAndGet();
- return;
- }
-
- Utils.checkState(server == null, "Server already running but ref count is 0.");
- if (server == null) {
- try {
- server = new RpcServer(config);
- } catch (InterruptedException ie) {
- throw Utils.propagate(ie);
- }
- }
-
- refCount.incrementAndGet();
- }
-
- synchronized void unref() {
- if (refCount.decrementAndGet() == 0) {
- server.close();
- server = null;
- }
- }
-
- private static Promise<ContextInfo> createContextInfo(final URI uri) {
- String[] userInfo = uri.getUserInfo().split(":", 2);
- ImmediateEventExecutor executor = ImmediateEventExecutor.INSTANCE;
- Promise<ContextInfo> promise = executor.newPromise();
- promise.setSuccess(new ContextInfo(uri.getHost(), uri.getPort(), userInfo[0], userInfo[1]));
- return promise;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java b/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java
deleted file mode 100644
index afd935d..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import java.io.IOException;
-import java.net.Inet4Address;
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-import javax.security.sasl.Sasl;
-
-import com.cloudera.livy.client.common.ClientConf;
-
-public class RSCConf extends ClientConf<RSCConf> {
-
- public static final String SPARK_CONF_PREFIX = "spark.";
- public static final String LIVY_SPARK_PREFIX = SPARK_CONF_PREFIX + "__livy__.";
-
- private static final String RSC_CONF_PREFIX = "livy.rsc.";
-
- public static enum Entry implements ConfEntry {
- CLIENT_ID("client.auth.id", null),
- CLIENT_SECRET("client.auth.secret", null),
- CLIENT_IN_PROCESS("client.do-not-use.run-driver-in-process", false),
- CLIENT_SHUTDOWN_TIMEOUT("client.shutdown-timeout", "10s"),
- DRIVER_CLASS("driver-class", null),
- SESSION_KIND("session.kind", null),
-
- LIVY_JARS("jars", null),
- SPARKR_PACKAGE("sparkr.package", null),
- PYSPARK_ARCHIVES("pyspark.archives", null),
-
- // Address for the RSC driver to connect back with it's connection info.
- LAUNCHER_ADDRESS("launcher.address", null),
- LAUNCHER_PORT_RANGE("launcher.port.range", "10000~10010"),
- // Setting up of this propety by user has no benefit. It is currently being used
- // to pass port information from ContextLauncher to RSCDriver
- LAUNCHER_PORT("launcher.port", -1),
- // How long will the RSC wait for a connection for a Livy server before shutting itself down.
- SERVER_IDLE_TIMEOUT("server.idle-timeout", "10m"),
-
- PROXY_USER("proxy-user", null),
-
- RPC_SERVER_ADDRESS("rpc.server.address", null),
- RPC_CLIENT_HANDSHAKE_TIMEOUT("server.connect.timeout", "90s"),
- RPC_CLIENT_CONNECT_TIMEOUT("client.connect.timeout", "10s"),
- RPC_CHANNEL_LOG_LEVEL("channel.log.level", null),
- RPC_MAX_MESSAGE_SIZE("rpc.max.size", 50 * 1024 * 1024),
- RPC_MAX_THREADS("rpc.threads", 8),
- RPC_SECRET_RANDOM_BITS("secret.bits", 256),
-
- SASL_MECHANISMS("rpc.sasl.mechanisms", "DIGEST-MD5"),
- SASL_QOP("rpc.sasl.qop", null),
-
- TEST_STUCK_END_SESSION("test.do-not-use.stuck-end-session", false),
- TEST_STUCK_START_DRIVER("test.do-not-use.stuck-start-driver", false),
-
- JOB_CANCEL_TRIGGER_INTERVAL("job-cancel.trigger-interval", "100ms"),
- JOB_CANCEL_TIMEOUT("job-cancel.timeout", "30s"),
-
- RETAINED_STATEMENT_NUMBER("retained-statements", 100);
-
- private final String key;
- private final Object dflt;
-
- private Entry(String key, Object dflt) {
- this.key = RSC_CONF_PREFIX + key;
- this.dflt = dflt;
- }
-
- @Override
- public String key() { return key; }
-
- @Override
- public Object dflt() { return dflt; }
- }
-
- public RSCConf() {
- this(new Properties());
- }
-
- public RSCConf(Properties config) {
- super(config);
- }
-
- public Map<String, String> getSaslOptions() {
- Map<String, String> opts = new HashMap<>();
-
- // TODO: add more options?
- String qop = get(Entry.SASL_QOP);
- if (qop != null) {
- opts.put(Sasl.QOP, qop);
- }
-
- return opts;
- }
-
- public String findLocalAddress() throws IOException {
- InetAddress address = InetAddress.getLocalHost();
- if (address.isLoopbackAddress()) {
- // Address resolves to something like 127.0.1.1, which happens on Debian;
- // try to find a better address using the local network interfaces
- Enumeration<NetworkInterface> ifaces = NetworkInterface.getNetworkInterfaces();
- while (ifaces.hasMoreElements()) {
- NetworkInterface ni = ifaces.nextElement();
- Enumeration<InetAddress> addrs = ni.getInetAddresses();
- while (addrs.hasMoreElements()) {
- InetAddress addr = addrs.nextElement();
- if (!addr.isLinkLocalAddress() && !addr.isLoopbackAddress()
- && addr instanceof Inet4Address) {
- // We've found an address that looks reasonable!
- LOG.warn("Your hostname, {}, resolves to a loopback address; using {} "
- + " instead (on interface {})", address.getHostName(), addr.getHostAddress(),
- ni.getName());
- LOG.warn("Set '{}' if you need to bind to another address.",
- Entry.RPC_SERVER_ADDRESS.key);
- return addr.getHostAddress();
- }
- }
- }
- }
-
- LOG.warn("Your hostname, {}, resolves to a loopback address, but we couldn't find "
- + "any external IP address!", address.getCanonicalHostName());
- LOG.warn("Set {} if you need to bind to another address.",
- Entry.RPC_SERVER_ADDRESS.key);
- return address.getCanonicalHostName();
- }
-
- private static final Map<String, DeprecatedConf> configsWithAlternatives
- = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{
- put(RSCConf.Entry.CLIENT_IN_PROCESS.key, DepConf.CLIENT_IN_PROCESS);
- put(RSCConf.Entry.CLIENT_SHUTDOWN_TIMEOUT.key, DepConf.CLIENT_SHUTDOWN_TIMEOUT);
- put(RSCConf.Entry.DRIVER_CLASS.key, DepConf.DRIVER_CLASS);
- put(RSCConf.Entry.SERVER_IDLE_TIMEOUT.key, DepConf.SERVER_IDLE_TIMEOUT);
- put(RSCConf.Entry.PROXY_USER.key, DepConf.PROXY_USER);
- put(RSCConf.Entry.TEST_STUCK_END_SESSION.key, DepConf.TEST_STUCK_END_SESSION);
- put(RSCConf.Entry.TEST_STUCK_START_DRIVER.key, DepConf.TEST_STUCK_START_DRIVER);
- put(RSCConf.Entry.JOB_CANCEL_TRIGGER_INTERVAL.key, DepConf.JOB_CANCEL_TRIGGER_INTERVAL);
- put(RSCConf.Entry.JOB_CANCEL_TIMEOUT.key, DepConf.JOB_CANCEL_TIMEOUT);
- put(RSCConf.Entry.RETAINED_STATEMENT_NUMBER.key, DepConf.RETAINED_STATEMENT_NUMBER);
- }});
-
- // Maps deprecated key to DeprecatedConf with the same key.
- // There are no deprecated configs without alternatives currently.
- private static final Map<String, DeprecatedConf> deprecatedConfigs
- = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>());
-
- protected Map<String, DeprecatedConf> getConfigsWithAlternatives() {
- return configsWithAlternatives;
- }
-
- protected Map<String, DeprecatedConf> getDeprecatedConfigs() {
- return deprecatedConfigs;
- }
-
- static enum DepConf implements DeprecatedConf {
- CLIENT_IN_PROCESS("client.do_not_use.run_driver_in_process", "0.4"),
- CLIENT_SHUTDOWN_TIMEOUT("client.shutdown_timeout", "0.4"),
- DRIVER_CLASS("driver_class", "0.4"),
- SERVER_IDLE_TIMEOUT("server.idle_timeout", "0.4"),
- PROXY_USER("proxy_user", "0.4"),
- TEST_STUCK_END_SESSION("test.do_not_use.stuck_end_session", "0.4"),
- TEST_STUCK_START_DRIVER("test.do_not_use.stuck_start_driver", "0.4"),
- JOB_CANCEL_TRIGGER_INTERVAL("job_cancel.trigger_interval", "0.4"),
- JOB_CANCEL_TIMEOUT("job_cancel.timeout", "0.4"),
- RETAINED_STATEMENT_NUMBER("retained_statements", "0.4");
-
- private final String key;
- private final String version;
- private final String deprecationMessage;
-
- private DepConf(String key, String version) {
- this(key, version, "");
- }
-
- private DepConf(String key, String version, String deprecationMessage) {
- this.key = RSC_CONF_PREFIX + key;
- this.version = version;
- this.deprecationMessage = deprecationMessage;
- }
-
- @Override
- public String key() { return key; }
-
- @Override
- public String version() { return version; }
-
- @Override
- public String deprecationMessage() { return deprecationMessage; }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/ReplJobResults.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/ReplJobResults.java b/rsc/src/main/java/com/cloudera/livy/rsc/ReplJobResults.java
deleted file mode 100644
index deef162..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/ReplJobResults.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.rsc;
-
-import com.cloudera.livy.rsc.driver.Statement;
-
-public class ReplJobResults {
- public final Statement[] statements;
-
- public ReplJobResults(Statement[] statements) {
- this.statements = statements;
- }
-
- public ReplJobResults() {
- this(null);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/Utils.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/Utils.java b/rsc/src/main/java/com/cloudera/livy/rsc/Utils.java
deleted file mode 100644
index 993ea1e..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/Utils.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import io.netty.util.concurrent.Future;
-import io.netty.util.concurrent.GenericFutureListener;
-
-/**
- * A few simple utility functions used by the code, mostly to avoid a direct dependency
- * on Guava.
- */
-public class Utils {
-
- public static void checkArgument(boolean condition) {
- if (!condition) {
- throw new IllegalArgumentException();
- }
- }
-
- public static void checkArgument(boolean condition, String msg, Object... args) {
- if (!condition) {
- throw new IllegalArgumentException(String.format(msg, args));
- }
- }
-
- public static void checkState(boolean condition, String msg, Object... args) {
- if (!condition) {
- throw new IllegalStateException(String.format(msg, args));
- }
- }
-
- public static void checkNotNull(Object o) {
- if (o == null) {
- throw new NullPointerException();
- }
- }
-
- public static RuntimeException propagate(Throwable t) {
- if (t instanceof RuntimeException) {
- throw (RuntimeException) t;
- } else {
- throw new RuntimeException(t);
- }
- }
-
- public static ThreadFactory newDaemonThreadFactory(final String nameFormat) {
- return new ThreadFactory() {
-
- private final AtomicInteger threadId = new AtomicInteger();
-
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new Thread(r);
- t.setName(String.format(nameFormat, threadId.incrementAndGet()));
- t.setDaemon(true);
- return t;
- }
-
- };
- }
-
- public static String join(Iterable<String> strs, String sep) {
- StringBuilder sb = new StringBuilder();
- for (String s : strs) {
- if (s != null && !s.isEmpty()) {
- sb.append(s).append(sep);
- }
- }
- if (sb.length() > 0) {
- sb.setLength(sb.length() - sep.length());
- }
- return sb.toString();
- }
-
- public static String stackTraceAsString(Throwable t) {
- StringBuilder sb = new StringBuilder();
- sb.append(t.getClass().getName()).append(": ").append(t.getMessage());
- for (StackTraceElement e : t.getStackTrace()) {
- sb.append("\n");
- sb.append(e.toString());
- }
- return sb.toString();
- }
-
- public static <T> void addListener(Future<T> future, final FutureListener<T> lsnr) {
- future.addListener(new GenericFutureListener<Future<T>>() {
- @Override
- public void operationComplete(Future<T> f) throws Exception {
- if (f.isSuccess()) {
- lsnr.onSuccess(f.get());
- } else {
- lsnr.onFailure(f.cause());
- }
- }
- });
- }
-
- private Utils() { }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/AddFileJob.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/AddFileJob.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/AddFileJob.java
deleted file mode 100644
index 82b0a69..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/AddFileJob.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.rsc.driver;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class AddFileJob implements Job<Object> {
-
- private final String path;
-
- AddFileJob() {
- this(null);
-}
-
- public AddFileJob(String path) {
- this.path = path;
-}
-
- @Override
- public Object call(JobContext jc) throws Exception {
- JobContextImpl jobContextImpl = (JobContextImpl)jc;
- jobContextImpl.addFile(path);
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/AddJarJob.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/AddJarJob.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/AddJarJob.java
deleted file mode 100644
index d23f3b8..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/AddJarJob.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.driver;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class AddJarJob implements Job<Object> {
-
- private final String path;
-
- // For serialization.
- private AddJarJob() {
- this(null);
- }
-
- public AddJarJob(String path) {
- this.path = path;
- }
-
- @Override
- public Object call(JobContext jc) throws Exception {
- JobContextImpl jobContextImpl = (JobContextImpl)jc;
- jobContextImpl.addJarOrPyFile(path);
- return null;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/BypassJob.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/BypassJob.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/BypassJob.java
deleted file mode 100644
index 8ca2c8e..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/BypassJob.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.driver;
-
-import java.nio.ByteBuffer;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-import com.cloudera.livy.client.common.BufferUtils;
-import com.cloudera.livy.client.common.Serializer;
-
-class BypassJob implements Job<byte[]> {
-
- private final Serializer serializer;
- private final byte[] serializedJob;
-
- BypassJob(Serializer serializer, byte[] serializedJob) {
- this.serializer = serializer;
- this.serializedJob = serializedJob;
- }
-
- @Override
- public byte[] call(JobContext jc) throws Exception {
- Job<?> job = (Job<?>) serializer.deserialize(ByteBuffer.wrap(serializedJob));
- Object result = job.call(jc);
- byte[] serializedResult;
- if (result != null) {
- ByteBuffer data = serializer.serialize(result);
- serializedResult = BufferUtils.toByteArray(data);
- } else {
- serializedResult = null;
- }
- return serializedResult;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/BypassJobWrapper.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/BypassJobWrapper.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/BypassJobWrapper.java
deleted file mode 100644
index b8951af..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/BypassJobWrapper.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.driver;
-
-import java.util.List;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobHandle;
-import com.cloudera.livy.rsc.BypassJobStatus;
-import com.cloudera.livy.rsc.Utils;
-
-public class BypassJobWrapper extends JobWrapper<byte[]> {
-
- private volatile byte[] result;
- private volatile Throwable error;
- private volatile JobHandle.State state;
- private volatile List<Integer> newSparkJobs;
-
- public BypassJobWrapper(RSCDriver driver, String jobId, Job<byte[]> serializedJob) {
- super(driver, jobId, serializedJob);
- state = JobHandle.State.QUEUED;
- }
-
- @Override
- public Void call() throws Exception {
- state = JobHandle.State.STARTED;
- return super.call();
- }
-
- @Override
- protected synchronized void finished(byte[] result, Throwable error) {
- if (error == null) {
- this.result = result;
- this.state = JobHandle.State.SUCCEEDED;
- } else {
- this.error = error;
- this.state = JobHandle.State.FAILED;
- }
- }
-
- @Override
- boolean cancel() {
- if (super.cancel()) {
- this.state = JobHandle.State.CANCELLED;
- return true;
- }
- return false;
- }
-
- @Override
- protected void jobStarted() {
- // Do nothing; just avoid sending data back to the driver.
- }
-
- synchronized BypassJobStatus getStatus() {
- String stackTrace = error != null ? Utils.stackTraceAsString(error) : null;
- return new BypassJobStatus(state, result, stackTrace);
- }
-
-}
[30/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/test/scala/com/cloudera/livy/client/http/LivyConnectionSpec.scala
----------------------------------------------------------------------
diff --git a/client-http/src/test/scala/com/cloudera/livy/client/http/LivyConnectionSpec.scala b/client-http/src/test/scala/com/cloudera/livy/client/http/LivyConnectionSpec.scala
deleted file mode 100644
index e1d29ca..0000000
--- a/client-http/src/test/scala/com/cloudera/livy/client/http/LivyConnectionSpec.scala
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.client.http
-
-import java.io.IOException
-import java.net.URLEncoder
-import java.nio.charset.StandardCharsets.UTF_8
-
-import org.apache.http.client.utils.URIBuilder
-import org.eclipse.jetty.security._
-import org.eclipse.jetty.security.authentication.BasicAuthenticator
-import org.eclipse.jetty.util.security._
-import org.scalatest.{BeforeAndAfterAll, FunSpecLike}
-import org.scalatest.Matchers._
-import org.scalatra.servlet.ScalatraListener
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf}
-import com.cloudera.livy.server.WebServer
-
-class LivyConnectionSpec extends FunSpecLike with BeforeAndAfterAll with LivyBaseUnitTestSuite {
- describe("LivyConnection") {
- def basicAuth(username: String, password: String, realm: String): SecurityHandler = {
- val roles = Array("user")
-
- val l = new HashLoginService()
- l.putUser(username, Credential.getCredential(password), roles)
- l.setName(realm)
-
- val constraint = new Constraint()
- constraint.setName(Constraint.__BASIC_AUTH)
- constraint.setRoles(roles)
- constraint.setAuthenticate(true)
-
- val cm = new ConstraintMapping()
- cm.setConstraint(constraint)
- cm.setPathSpec("/*")
-
- val csh = new ConstraintSecurityHandler()
- csh.setAuthenticator(new BasicAuthenticator())
- csh.setRealmName(realm)
- csh.addConstraintMapping(cm)
- csh.setLoginService(l)
-
- csh
- }
-
- def test(password: String, livyConf: LivyConf = new LivyConf()): Unit = {
- val username = "user name"
-
- val server = new WebServer(livyConf, "0.0.0.0", 0)
- server.context.setSecurityHandler(basicAuth(username, password, "realm"))
- server.context.setResourceBase("src/main/com/cloudera/livy/server")
- server.context.setInitParameter(ScalatraListener.LifeCycleKey,
- classOf[HttpClientTestBootstrap].getCanonicalName)
- server.context.addEventListener(new ScalatraListener)
- server.start()
-
- val utf8Name = UTF_8.name()
- val uri = new URIBuilder()
- .setScheme(server.protocol)
- .setHost(server.host)
- .setPort(server.port)
- .setUserInfo(URLEncoder.encode(username, utf8Name), URLEncoder.encode(password, utf8Name))
- .build()
- info(uri.toString)
- val conn = new LivyConnection(uri, new HttpConf(null))
- try {
- conn.get(classOf[Object], "/") should not be (null)
-
- } finally {
- conn.close()
- }
-
- server.stop()
- server.join()
- }
-
- it("should support HTTP auth with password") {
- test("pass:word")
- }
-
- it("should support HTTP auth with empty password") {
- test("")
- }
-
- it("should be failed with large header size") {
- val livyConf = new LivyConf()
- .set(LivyConf.REQUEST_HEADER_SIZE, 1024)
- .set(LivyConf.RESPONSE_HEADER_SIZE, 1024)
- val pwd = "test-password" * 100
- val exception = intercept[IOException](test(pwd, livyConf))
- exception.getMessage.contains("Request Entity Too Large") should be(true)
- }
-
- it("should be succeeded with configured header size") {
- val livyConf = new LivyConf()
- .set(LivyConf.REQUEST_HEADER_SIZE, 2048)
- .set(LivyConf.RESPONSE_HEADER_SIZE, 2048)
- val pwd = "test-password" * 100
- test(pwd, livyConf)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/test/scala/org/apache/livy/client/http/HttpClientSpec.scala
----------------------------------------------------------------------
diff --git a/client-http/src/test/scala/org/apache/livy/client/http/HttpClientSpec.scala b/client-http/src/test/scala/org/apache/livy/client/http/HttpClientSpec.scala
new file mode 100644
index 0000000..801c09b
--- /dev/null
+++ b/client-http/src/test/scala/org/apache/livy/client/http/HttpClientSpec.scala
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.http
+
+import java.io.{File, InputStream}
+import java.net.{InetAddress, URI}
+import java.nio.file.{Files, Paths}
+import java.util.concurrent.{Future => JFuture, _}
+import java.util.concurrent.atomic.AtomicLong
+import javax.servlet.ServletContext
+import javax.servlet.http.HttpServletRequest
+
+import scala.concurrent.{ExecutionContext, Future}
+
+import org.mockito.ArgumentCaptor
+import org.mockito.Matchers.{eq => meq, _}
+import org.mockito.Mockito._
+import org.scalatest.{BeforeAndAfterAll, FunSpecLike}
+import org.scalatra.LifeCycle
+import org.scalatra.servlet.ScalatraListener
+
+import org.apache.livy._
+import org.apache.livy.client.common.{BufferUtils, Serializer}
+import org.apache.livy.client.common.HttpMessages._
+import org.apache.livy.server.WebServer
+import org.apache.livy.server.interactive.{InteractiveSession, InteractiveSessionServlet}
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.{InteractiveSessionManager, SessionState, Spark}
+import org.apache.livy.test.jobs.Echo
+import org.apache.livy.utils.AppInfo
+
+/**
+ * The test for the HTTP client is written in Scala so we can reuse the code in the livy-server
+ * module, which implements the client session backend. The client servlet has some functionality
+ * overridden to avoid creating sub-processes for each seession.
+ */
+class HttpClientSpec extends FunSpecLike with BeforeAndAfterAll with LivyBaseUnitTestSuite {
+
+ import HttpClientSpec._
+
+ private val TIMEOUT_S = 10
+ private val ID_GENERATOR = new AtomicLong()
+ private val serializer = new Serializer()
+
+ private var server: WebServer = _
+ private var client: LivyClient = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ server = new WebServer(new LivyConf(), "0.0.0.0", 0)
+
+ server.context.setResourceBase("src/main/com/cloudera/livy/server")
+ server.context.setInitParameter(ScalatraListener.LifeCycleKey,
+ classOf[HttpClientTestBootstrap].getCanonicalName)
+ server.context.addEventListener(new ScalatraListener)
+
+ server.start()
+ }
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ if (server != null) {
+ server.stop()
+ server = null
+ }
+ if (client != null) {
+ client.stop(true)
+ client = null
+ }
+ session = null
+ }
+
+ describe("HTTP client library") {
+
+ it("should create clients") {
+ // WebServer does this internally instead of respecting "0.0.0.0", so try to use the same
+ // address.
+ val uri = s"http://${InetAddress.getLocalHost.getHostAddress}:${server.port}/"
+ client = new LivyClientBuilder(false).setURI(new URI(uri)).build()
+ }
+
+ withClient("should run and monitor asynchronous jobs") {
+ testJob(false)
+ }
+
+ withClient("should propagate errors from jobs") {
+ val errorMessage = "This job throws an error."
+ val (jobId, handle) = runJob(false, { id => Seq(
+ new JobStatus(id, JobHandle.State.FAILED, null, errorMessage))
+ })
+
+ val error = intercept[ExecutionException] {
+ handle.get(TIMEOUT_S, TimeUnit.SECONDS)
+ }
+ assert(error.getCause() != null)
+ assert(error.getCause().getMessage().indexOf(errorMessage) >= 0)
+ verify(session, times(1)).jobStatus(meq(jobId))
+ }
+
+ withClient("should run and monitor synchronous jobs") {
+ testJob(false)
+ }
+
+ withClient("should add files and jars") {
+ val furi = new URI("hdfs:file")
+ val juri = new URI("hdfs:jar")
+
+ client.addFile(furi).get(TIMEOUT_S, TimeUnit.SECONDS)
+ client.addJar(juri).get(TIMEOUT_S, TimeUnit.SECONDS)
+
+ verify(session, times(1)).addFile(meq(furi))
+ verify(session, times(1)).addJar(meq(juri))
+ }
+
+ withClient("should upload files and jars") {
+ uploadAndVerify("file")
+ uploadAndVerify("jar")
+ }
+
+ withClient("should cancel jobs") {
+ val (jobId, handle) = runJob(false, { id => Seq(
+ new JobStatus(id, JobHandle.State.STARTED, null, null),
+ new JobStatus(id, JobHandle.State.CANCELLED, null, null))
+ })
+ handle.cancel(true)
+
+ intercept[CancellationException] {
+ handle.get(TIMEOUT_S, TimeUnit.SECONDS)
+ }
+
+ verify(session, times(1)).cancelJob(meq(jobId))
+ }
+
+ withClient("should notify listeners of job completion") {
+ val (jobId, handle) = runJob(false, { id => Seq(
+ new JobStatus(id, JobHandle.State.STARTED, null, null),
+ new JobStatus(id, JobHandle.State.SUCCEEDED, serialize(id), null))
+ })
+
+ val listener = mock(classOf[JobHandle.Listener[Long]])
+ handle.asInstanceOf[JobHandle[Long]].addListener(listener)
+
+ assert(handle.get(TIMEOUT_S, TimeUnit.SECONDS) === jobId)
+ verify(listener, times(1)).onJobSucceeded(any(), any())
+ }
+
+ withClient("should time out handle get() call") {
+ // JobHandleImpl does exponential backoff checking the result of a job. Given an initial
+ // wait of 100ms, 4 iterations should result in a wait of 800ms, so the handle should at that
+ // point timeout a wait of 100ms.
+ val (jobId, handle) = runJob(false, { id => Seq(
+ new JobStatus(id, JobHandle.State.STARTED, null, null),
+ new JobStatus(id, JobHandle.State.STARTED, null, null),
+ new JobStatus(id, JobHandle.State.STARTED, null, null),
+ new JobStatus(id, JobHandle.State.SUCCEEDED, serialize(id), null))
+ })
+
+ intercept[TimeoutException] {
+ handle.get(100, TimeUnit.MILLISECONDS)
+ }
+
+ assert(handle.get(TIMEOUT_S, TimeUnit.SECONDS) === jobId)
+ }
+
+ withClient("should handle null responses") {
+ testJob(false, response = Some(null))
+ }
+
+ withClient("should connect to existing sessions") {
+ var sid = client.asInstanceOf[HttpClient].getSessionId()
+ val uri = s"http://${InetAddress.getLocalHost.getHostAddress}:${server.port}" +
+ s"${LivyConnection.SESSIONS_URI}/$sid"
+ val newClient = new LivyClientBuilder(false).setURI(new URI(uri)).build()
+ newClient.stop(false)
+ verify(session, never()).stop()
+ }
+
+ withClient("should tear down clients") {
+ client.stop(true)
+ verify(session, times(1)).stop()
+ }
+
+ }
+
+ private def uploadAndVerify(cmd: String): Unit = {
+ val f = File.createTempFile("uploadTestFile", cmd)
+ val expectedStr = "Test data"
+ val expectedData = expectedStr.getBytes()
+ Files.write(Paths.get(f.getAbsolutePath), expectedData)
+ val b = new Array[Byte](expectedData.length)
+ val captor = ArgumentCaptor.forClass(classOf[InputStream])
+ if (cmd == "file") {
+ client.uploadFile(f).get(TIMEOUT_S, TimeUnit.SECONDS)
+ verify(session, times(1)).addFile(captor.capture(), meq(f.getName))
+ } else {
+ client.uploadJar(f).get(TIMEOUT_S, TimeUnit.SECONDS)
+ verify(session, times(1)).addJar(captor.capture(), meq(f.getName))
+ }
+ captor.getValue.read(b)
+ assert(expectedStr === new String(b))
+ }
+
+ private def runJob(sync: Boolean, genStatusFn: Long => Seq[JobStatus]): (Long, JFuture[Int]) = {
+ val jobId = java.lang.Long.valueOf(ID_GENERATOR.incrementAndGet())
+ when(session.submitJob(any(classOf[Array[Byte]]))).thenReturn(jobId)
+
+ val statuses = genStatusFn(jobId)
+ val first = statuses.head
+ val remaining = statuses.drop(1)
+ when(session.jobStatus(meq(jobId))).thenReturn(first, remaining: _*)
+
+ val job = new Echo(42)
+ val handle = if (sync) client.run(job) else client.submit(job)
+ (jobId, handle)
+ }
+
+ private def testJob(sync: Boolean, response: Option[Any] = None): Unit = {
+ val (jobId, handle) = runJob(sync, { id => Seq(
+ new JobStatus(id, JobHandle.State.STARTED, null, null),
+ new JobStatus(id, JobHandle.State.SUCCEEDED, serialize(response.getOrElse(id)), null))
+ })
+ assert(handle.get(TIMEOUT_S, TimeUnit.SECONDS) === response.getOrElse(jobId))
+ verify(session, times(2)).jobStatus(meq(jobId))
+ }
+
+ private def withClient(desc: String)(fn: => Unit): Unit = {
+ it(desc) {
+ assume(client != null, "No active client.")
+ fn
+ }
+ }
+
+ def serialize(value: Any): Array[Byte] = {
+ BufferUtils.toByteArray(serializer.serialize(value))
+ }
+
+}
+
+private object HttpClientSpec {
+
+ // Hack warning: keep the session object available so that individual tests can mock
+ // the desired behavior before making requests to the server.
+ var session: InteractiveSession = _
+
+}
+
+private class HttpClientTestBootstrap extends LifeCycle {
+
+ private implicit def executor: ExecutionContext = ExecutionContext.global
+
+ override def init(context: ServletContext): Unit = {
+ val conf = new LivyConf()
+ val stateStore = mock(classOf[SessionStore])
+ val sessionManager = new InteractiveSessionManager(conf, stateStore, Some(Seq.empty))
+ val servlet = new InteractiveSessionServlet(sessionManager, stateStore, conf) {
+ override protected def createSession(req: HttpServletRequest): InteractiveSession = {
+ val session = mock(classOf[InteractiveSession])
+ val id = sessionManager.nextId()
+ when(session.id).thenReturn(id)
+ when(session.appId).thenReturn(None)
+ when(session.appInfo).thenReturn(AppInfo())
+ when(session.state).thenReturn(SessionState.Idle())
+ when(session.proxyUser).thenReturn(None)
+ when(session.kind).thenReturn(Spark())
+ when(session.stop()).thenReturn(Future.successful(()))
+ require(HttpClientSpec.session == null, "Session already created?")
+ HttpClientSpec.session = session
+ session
+ }
+ }
+
+ context.mount(servlet, s"${LivyConnection.SESSIONS_URI}/*")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-http/src/test/scala/org/apache/livy/client/http/LivyConnectionSpec.scala
----------------------------------------------------------------------
diff --git a/client-http/src/test/scala/org/apache/livy/client/http/LivyConnectionSpec.scala b/client-http/src/test/scala/org/apache/livy/client/http/LivyConnectionSpec.scala
new file mode 100644
index 0000000..41edd66
--- /dev/null
+++ b/client-http/src/test/scala/org/apache/livy/client/http/LivyConnectionSpec.scala
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.client.http
+
+import java.io.IOException
+import java.net.URLEncoder
+import java.nio.charset.StandardCharsets.UTF_8
+
+import org.apache.http.client.utils.URIBuilder
+import org.eclipse.jetty.security._
+import org.eclipse.jetty.security.authentication.BasicAuthenticator
+import org.eclipse.jetty.util.security._
+import org.scalatest.{BeforeAndAfterAll, FunSpecLike}
+import org.scalatest.Matchers._
+import org.scalatra.servlet.ScalatraListener
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+import org.apache.livy.server.WebServer
+
+class LivyConnectionSpec extends FunSpecLike with BeforeAndAfterAll with LivyBaseUnitTestSuite {
+ describe("LivyConnection") {
+ def basicAuth(username: String, password: String, realm: String): SecurityHandler = {
+ val roles = Array("user")
+
+ val l = new HashLoginService()
+ l.putUser(username, Credential.getCredential(password), roles)
+ l.setName(realm)
+
+ val constraint = new Constraint()
+ constraint.setName(Constraint.__BASIC_AUTH)
+ constraint.setRoles(roles)
+ constraint.setAuthenticate(true)
+
+ val cm = new ConstraintMapping()
+ cm.setConstraint(constraint)
+ cm.setPathSpec("/*")
+
+ val csh = new ConstraintSecurityHandler()
+ csh.setAuthenticator(new BasicAuthenticator())
+ csh.setRealmName(realm)
+ csh.addConstraintMapping(cm)
+ csh.setLoginService(l)
+
+ csh
+ }
+
+ def test(password: String, livyConf: LivyConf = new LivyConf()): Unit = {
+ val username = "user name"
+
+ val server = new WebServer(livyConf, "0.0.0.0", 0)
+ server.context.setSecurityHandler(basicAuth(username, password, "realm"))
+ server.context.setResourceBase("src/main/com/cloudera/livy/server")
+ server.context.setInitParameter(ScalatraListener.LifeCycleKey,
+ classOf[HttpClientTestBootstrap].getCanonicalName)
+ server.context.addEventListener(new ScalatraListener)
+ server.start()
+
+ val utf8Name = UTF_8.name()
+ val uri = new URIBuilder()
+ .setScheme(server.protocol)
+ .setHost(server.host)
+ .setPort(server.port)
+ .setUserInfo(URLEncoder.encode(username, utf8Name), URLEncoder.encode(password, utf8Name))
+ .build()
+ info(uri.toString)
+ val conn = new LivyConnection(uri, new HttpConf(null))
+ try {
+ conn.get(classOf[Object], "/") should not be (null)
+
+ } finally {
+ conn.close()
+ }
+
+ server.stop()
+ server.join()
+ }
+
+ it("should support HTTP auth with password") {
+ test("pass:word")
+ }
+
+ it("should support HTTP auth with empty password") {
+ test("")
+ }
+
+ it("should be failed with large header size") {
+ val livyConf = new LivyConf()
+ .set(LivyConf.REQUEST_HEADER_SIZE, 1024)
+ .set(LivyConf.RESPONSE_HEADER_SIZE, 1024)
+ val pwd = "test-password" * 100
+ val exception = intercept[IOException](test(pwd, livyConf))
+ exception.getMessage.contains("Request Entity Too Large") should be(true)
+ }
+
+ it("should be succeeded with configured header size") {
+ val livyConf = new LivyConf()
+ .set(LivyConf.REQUEST_HEADER_SIZE, 2048)
+ .set(LivyConf.RESPONSE_HEADER_SIZE, 2048)
+ val pwd = "test-password" * 100
+ test(pwd, livyConf)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index e08e55b..0a0344b 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -20,19 +20,19 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>multi-scala-project-root</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../scala/pom.xml</relativePath>
</parent>
<artifactId>livy-core-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<dependencies>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-client-common</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/scala-2.10/pom.xml
----------------------------------------------------------------------
diff --git a/core/scala-2.10/pom.xml b/core/scala-2.10/pom.xml
index b8e7bc5..8d1aad8 100644
--- a/core/scala-2.10/pom.xml
+++ b/core/scala-2.10/pom.xml
@@ -17,15 +17,15 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-core_2.10</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-core-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/scala-2.11/pom.xml
----------------------------------------------------------------------
diff --git a/core/scala-2.11/pom.xml b/core/scala-2.11/pom.xml
index 1f817ab..bc16caf 100644
--- a/core/scala-2.11/pom.xml
+++ b/core/scala-2.11/pom.xml
@@ -17,15 +17,15 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-core_2.11</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-core-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/main/scala/com/cloudera/livy/Logging.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/com/cloudera/livy/Logging.scala b/core/src/main/scala/com/cloudera/livy/Logging.scala
deleted file mode 100644
index 8532ea2..0000000
--- a/core/src/main/scala/com/cloudera/livy/Logging.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy
-
-import org.slf4j.LoggerFactory
-
-trait Logging {
- lazy val logger = LoggerFactory.getLogger(this.getClass)
-
- def trace(message: => Any): Unit = {
- if (logger.isTraceEnabled) {
- logger.trace(message.toString)
- }
- }
-
- def debug(message: => Any): Unit = {
- if (logger.isDebugEnabled) {
- logger.debug(message.toString)
- }
- }
-
- def info(message: => Any): Unit = {
- if (logger.isInfoEnabled) {
- logger.info(message.toString)
- }
- }
-
- def warn(message: => Any): Unit = {
- logger.warn(message.toString)
- }
-
- def error(message: => Any, t: Throwable): Unit = {
- logger.error(message.toString, t)
- }
-
- def error(message: => Any): Unit = {
- logger.error(message.toString)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/main/scala/com/cloudera/livy/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/com/cloudera/livy/Utils.scala b/core/src/main/scala/com/cloudera/livy/Utils.scala
deleted file mode 100644
index 0d779dd..0000000
--- a/core/src/main/scala/com/cloudera/livy/Utils.scala
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy
-
-import java.io.{Closeable, File, FileInputStream, InputStreamReader}
-import java.net.URL
-import java.nio.charset.StandardCharsets.UTF_8
-import java.util.Properties
-
-import scala.annotation.tailrec
-import scala.collection.JavaConverters._
-import scala.concurrent.TimeoutException
-import scala.concurrent.duration.Duration
-
-object Utils {
- def getPropertiesFromFile(file: File): Map[String, String] = {
- loadProperties(file.toURI().toURL())
- }
-
- def loadProperties(url: URL): Map[String, String] = {
- val inReader = new InputStreamReader(url.openStream(), UTF_8)
- try {
- val properties = new Properties()
- properties.load(inReader)
- properties.stringPropertyNames().asScala.map { k =>
- (k, properties.getProperty(k).trim())
- }.toMap
- } finally {
- inReader.close()
- }
- }
-
- /**
- * Checks if event has occurred during some time period. This performs an exponential backoff
- * to limit the poll calls.
- *
- * @param checkForEvent
- * @param atMost
- * @throws java.util.concurrent.TimeoutException
- * @throws java.lang.InterruptedException
- * @return
- */
- @throws(classOf[TimeoutException])
- @throws(classOf[InterruptedException])
- final def waitUntil(checkForEvent: () => Boolean, atMost: Duration): Unit = {
- val endTime = System.currentTimeMillis() + atMost.toMillis
-
- @tailrec
- def aux(count: Int): Unit = {
- if (!checkForEvent()) {
- val now = System.currentTimeMillis()
-
- if (now < endTime) {
- val sleepTime = Math.max(10 * (2 << (count - 1)), 1000)
- Thread.sleep(sleepTime)
- aux(count + 1)
- } else {
- throw new TimeoutException
- }
- }
- }
-
- aux(1)
- }
-
- /** Returns if the process is still running */
- def isProcessAlive(process: Process): Boolean = {
- try {
- process.exitValue()
- false
- } catch {
- case _: IllegalThreadStateException =>
- true
- }
- }
-
- def startDaemonThread(name: String)(f: => Unit): Thread = {
- val thread = new Thread(name) {
- override def run(): Unit = f
- }
- thread.setDaemon(true)
- thread.start()
- thread
- }
-
- def usingResource[A <: Closeable, B](resource: A)(f: A => B): B = {
- try {
- f(resource)
- } finally {
- resource.close()
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/main/scala/com/cloudera/livy/msgs.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/com/cloudera/livy/msgs.scala b/core/src/main/scala/com/cloudera/livy/msgs.scala
deleted file mode 100644
index 39ee152..0000000
--- a/core/src/main/scala/com/cloudera/livy/msgs.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy
-
-sealed trait MsgType
-
-object MsgType {
- case object execute_request extends MsgType
- case object execute_reply extends MsgType
-}
-
-case class Msg[T <: Content](msg_type: MsgType, content: T)
-
-sealed trait Content
-
-case class ExecuteRequest(code: String) extends Content {
- val msg_type = MsgType.execute_request
-}
-
-sealed trait ExecutionStatus
-object ExecutionStatus {
- case object ok extends ExecutionStatus
- case object error extends ExecutionStatus
- case object abort extends ExecutionStatus
-}
-
-sealed trait ExecuteReply extends Content {
- val msg_type = MsgType.execute_reply
-
- val status: ExecutionStatus
- val execution_count: Int
-}
-
-case class ExecuteReplyOk(execution_count: Int,
- payload: Map[String, String]) extends ExecuteReply {
- val status = ExecutionStatus.ok
-}
-
-case class ExecuteReplyError(execution_count: Int,
- ename: String,
- evalue: String,
- traceback: List[String]) extends ExecuteReply {
- val status = ExecutionStatus.error
-}
-
-case class ExecuteResponse(id: Int, input: Seq[String], output: Seq[String])
-
-case class ShutdownRequest() extends Content
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/main/scala/com/cloudera/livy/sessions/Kind.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/com/cloudera/livy/sessions/Kind.scala b/core/src/main/scala/com/cloudera/livy/sessions/Kind.scala
deleted file mode 100644
index 7a6480b..0000000
--- a/core/src/main/scala/com/cloudera/livy/sessions/Kind.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.sessions
-
-import com.fasterxml.jackson.core.{JsonGenerator, JsonParser, JsonToken}
-import com.fasterxml.jackson.databind._
-import com.fasterxml.jackson.databind.module.SimpleModule
-
-sealed trait Kind
-case class Spark() extends Kind {
- override def toString: String = "spark"
-}
-
-case class PySpark() extends Kind {
- override def toString: String = "pyspark"
-}
-
-case class PySpark3() extends Kind {
- override def toString: String = "pyspark3"
-}
-
-case class SparkR() extends Kind {
- override def toString: String = "sparkr"
-}
-
-object Kind {
-
- def apply(kind: String): Kind = kind match {
- case "spark" | "scala" => Spark()
- case "pyspark" | "python" => PySpark()
- case "pyspark3" | "python3" => PySpark3()
- case "sparkr" | "r" => SparkR()
- case other => throw new IllegalArgumentException(s"Invalid kind: $other")
- }
-
-}
-
-class SessionKindModule extends SimpleModule("SessionKind") {
-
- addSerializer(classOf[Kind], new JsonSerializer[Kind]() {
- override def serialize(value: Kind, jgen: JsonGenerator, provider: SerializerProvider): Unit = {
- jgen.writeString(value.toString)
- }
- })
-
- addDeserializer(classOf[Kind], new JsonDeserializer[Kind]() {
- override def deserialize(jp: JsonParser, ctxt: DeserializationContext): Kind = {
- require(jp.getCurrentToken() == JsonToken.VALUE_STRING, "Kind should be a string.")
- Kind(jp.getText())
- }
- })
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/main/scala/com/cloudera/livy/sessions/SessionState.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/com/cloudera/livy/sessions/SessionState.scala b/core/src/main/scala/com/cloudera/livy/sessions/SessionState.scala
deleted file mode 100644
index 831b01a..0000000
--- a/core/src/main/scala/com/cloudera/livy/sessions/SessionState.scala
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.sessions
-
-sealed trait SessionState {
- /** Returns true if the State represents a process that can eventually execute commands */
- def isActive: Boolean
-}
-
-sealed trait FinishedSessionState extends SessionState {
- /** When session is finished. */
- def time: Long
-}
-
-object SessionState {
-
- def apply(s: String): SessionState = {
- s match {
- case "not_started" => NotStarted()
- case "starting" => Starting()
- case "recovering" => Recovering()
- case "idle" => Idle()
- case "running" => Running()
- case "busy" => Busy()
- case "shutting_down" => ShuttingDown()
- case "error" => Error()
- case "dead" => Dead()
- case "success" => Success()
- case _ => throw new IllegalArgumentException(s"Illegal session state: $s")
- }
- }
-
- case class NotStarted() extends SessionState {
- override def isActive: Boolean = true
-
- override def toString: String = "not_started"
- }
-
- case class Starting() extends SessionState {
- override def isActive: Boolean = true
-
- override def toString: String = "starting"
- }
-
- case class Recovering() extends SessionState {
- override def isActive: Boolean = true
-
- override def toString: String = "recovering"
- }
-
- case class Idle() extends SessionState {
- override def isActive: Boolean = true
-
- override def toString: String = "idle"
- }
-
- case class Running() extends SessionState {
- override def isActive: Boolean = true
-
- override def toString: String = "running"
- }
-
- case class Busy() extends SessionState {
- override def isActive: Boolean = true
-
- override def toString: String = "busy"
- }
-
- case class ShuttingDown() extends SessionState {
- override def isActive: Boolean = false
-
- override def toString: String = "shutting_down"
- }
-
- case class Error(time: Long = System.nanoTime()) extends FinishedSessionState {
- override def isActive: Boolean = true
-
- override def toString: String = "error"
- }
-
- case class Dead(time: Long = System.nanoTime()) extends FinishedSessionState {
- override def isActive: Boolean = false
-
- override def toString: String = "dead"
- }
-
- case class Success(time: Long = System.nanoTime()) extends FinishedSessionState {
- override def isActive: Boolean = false
-
- override def toString: String = "success"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/main/scala/org/apache/livy/Logging.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/livy/Logging.scala b/core/src/main/scala/org/apache/livy/Logging.scala
new file mode 100644
index 0000000..73ff7df
--- /dev/null
+++ b/core/src/main/scala/org/apache/livy/Logging.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy
+
+import org.slf4j.LoggerFactory
+
+trait Logging {
+ lazy val logger = LoggerFactory.getLogger(this.getClass)
+
+ def trace(message: => Any): Unit = {
+ if (logger.isTraceEnabled) {
+ logger.trace(message.toString)
+ }
+ }
+
+ def debug(message: => Any): Unit = {
+ if (logger.isDebugEnabled) {
+ logger.debug(message.toString)
+ }
+ }
+
+ def info(message: => Any): Unit = {
+ if (logger.isInfoEnabled) {
+ logger.info(message.toString)
+ }
+ }
+
+ def warn(message: => Any): Unit = {
+ logger.warn(message.toString)
+ }
+
+ def error(message: => Any, t: Throwable): Unit = {
+ logger.error(message.toString, t)
+ }
+
+ def error(message: => Any): Unit = {
+ logger.error(message.toString)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/main/scala/org/apache/livy/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/livy/Utils.scala b/core/src/main/scala/org/apache/livy/Utils.scala
new file mode 100644
index 0000000..c1cffe4
--- /dev/null
+++ b/core/src/main/scala/org/apache/livy/Utils.scala
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy
+
+import java.io.{Closeable, File, FileInputStream, InputStreamReader}
+import java.net.URL
+import java.nio.charset.StandardCharsets.UTF_8
+import java.util.Properties
+
+import scala.annotation.tailrec
+import scala.collection.JavaConverters._
+import scala.concurrent.TimeoutException
+import scala.concurrent.duration.Duration
+
+object Utils {
+ def getPropertiesFromFile(file: File): Map[String, String] = {
+ loadProperties(file.toURI().toURL())
+ }
+
+ def loadProperties(url: URL): Map[String, String] = {
+ val inReader = new InputStreamReader(url.openStream(), UTF_8)
+ try {
+ val properties = new Properties()
+ properties.load(inReader)
+ properties.stringPropertyNames().asScala.map { k =>
+ (k, properties.getProperty(k).trim())
+ }.toMap
+ } finally {
+ inReader.close()
+ }
+ }
+
+ /**
+ * Checks if event has occurred during some time period. This performs an exponential backoff
+ * to limit the poll calls.
+ *
+ * @param checkForEvent
+ * @param atMost
+ * @throws java.util.concurrent.TimeoutException
+ * @throws java.lang.InterruptedException
+ * @return
+ */
+ @throws(classOf[TimeoutException])
+ @throws(classOf[InterruptedException])
+ final def waitUntil(checkForEvent: () => Boolean, atMost: Duration): Unit = {
+ val endTime = System.currentTimeMillis() + atMost.toMillis
+
+ @tailrec
+ def aux(count: Int): Unit = {
+ if (!checkForEvent()) {
+ val now = System.currentTimeMillis()
+
+ if (now < endTime) {
+ val sleepTime = Math.max(10 * (2 << (count - 1)), 1000)
+ Thread.sleep(sleepTime)
+ aux(count + 1)
+ } else {
+ throw new TimeoutException
+ }
+ }
+ }
+
+ aux(1)
+ }
+
+ /** Returns if the process is still running */
+ def isProcessAlive(process: Process): Boolean = {
+ try {
+ process.exitValue()
+ false
+ } catch {
+ case _: IllegalThreadStateException =>
+ true
+ }
+ }
+
+ def startDaemonThread(name: String)(f: => Unit): Thread = {
+ val thread = new Thread(name) {
+ override def run(): Unit = f
+ }
+ thread.setDaemon(true)
+ thread.start()
+ thread
+ }
+
+ def usingResource[A <: Closeable, B](resource: A)(f: A => B): B = {
+ try {
+ f(resource)
+ } finally {
+ resource.close()
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/main/scala/org/apache/livy/msgs.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/livy/msgs.scala b/core/src/main/scala/org/apache/livy/msgs.scala
new file mode 100644
index 0000000..0dd0a26
--- /dev/null
+++ b/core/src/main/scala/org/apache/livy/msgs.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy
+
+sealed trait MsgType
+
+object MsgType {
+ case object execute_request extends MsgType
+ case object execute_reply extends MsgType
+}
+
+case class Msg[T <: Content](msg_type: MsgType, content: T)
+
+sealed trait Content
+
+case class ExecuteRequest(code: String) extends Content {
+ val msg_type = MsgType.execute_request
+}
+
+sealed trait ExecutionStatus
+object ExecutionStatus {
+ case object ok extends ExecutionStatus
+ case object error extends ExecutionStatus
+ case object abort extends ExecutionStatus
+}
+
+sealed trait ExecuteReply extends Content {
+ val msg_type = MsgType.execute_reply
+
+ val status: ExecutionStatus
+ val execution_count: Int
+}
+
+case class ExecuteReplyOk(execution_count: Int,
+ payload: Map[String, String]) extends ExecuteReply {
+ val status = ExecutionStatus.ok
+}
+
+case class ExecuteReplyError(execution_count: Int,
+ ename: String,
+ evalue: String,
+ traceback: List[String]) extends ExecuteReply {
+ val status = ExecutionStatus.error
+}
+
+case class ExecuteResponse(id: Int, input: Seq[String], output: Seq[String])
+
+case class ShutdownRequest() extends Content
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/main/scala/org/apache/livy/sessions/Kind.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/livy/sessions/Kind.scala b/core/src/main/scala/org/apache/livy/sessions/Kind.scala
new file mode 100644
index 0000000..bfb166f
--- /dev/null
+++ b/core/src/main/scala/org/apache/livy/sessions/Kind.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.sessions
+
+import com.fasterxml.jackson.core.{JsonGenerator, JsonParser, JsonToken}
+import com.fasterxml.jackson.databind._
+import com.fasterxml.jackson.databind.module.SimpleModule
+
+sealed trait Kind
+case class Spark() extends Kind {
+ override def toString: String = "spark"
+}
+
+case class PySpark() extends Kind {
+ override def toString: String = "pyspark"
+}
+
+case class PySpark3() extends Kind {
+ override def toString: String = "pyspark3"
+}
+
+case class SparkR() extends Kind {
+ override def toString: String = "sparkr"
+}
+
+object Kind {
+
+ def apply(kind: String): Kind = kind match {
+ case "spark" | "scala" => Spark()
+ case "pyspark" | "python" => PySpark()
+ case "pyspark3" | "python3" => PySpark3()
+ case "sparkr" | "r" => SparkR()
+ case other => throw new IllegalArgumentException(s"Invalid kind: $other")
+ }
+
+}
+
+class SessionKindModule extends SimpleModule("SessionKind") {
+
+ addSerializer(classOf[Kind], new JsonSerializer[Kind]() {
+ override def serialize(value: Kind, jgen: JsonGenerator, provider: SerializerProvider): Unit = {
+ jgen.writeString(value.toString)
+ }
+ })
+
+ addDeserializer(classOf[Kind], new JsonDeserializer[Kind]() {
+ override def deserialize(jp: JsonParser, ctxt: DeserializationContext): Kind = {
+ require(jp.getCurrentToken() == JsonToken.VALUE_STRING, "Kind should be a string.")
+ Kind(jp.getText())
+ }
+ })
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/main/scala/org/apache/livy/sessions/SessionState.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/livy/sessions/SessionState.scala b/core/src/main/scala/org/apache/livy/sessions/SessionState.scala
new file mode 100644
index 0000000..fd19321
--- /dev/null
+++ b/core/src/main/scala/org/apache/livy/sessions/SessionState.scala
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.sessions
+
+sealed trait SessionState {
+ /** Returns true if the State represents a process that can eventually execute commands */
+ def isActive: Boolean
+}
+
+sealed trait FinishedSessionState extends SessionState {
+ /** When session is finished. */
+ def time: Long
+}
+
+object SessionState {
+
+ def apply(s: String): SessionState = {
+ s match {
+ case "not_started" => NotStarted()
+ case "starting" => Starting()
+ case "recovering" => Recovering()
+ case "idle" => Idle()
+ case "running" => Running()
+ case "busy" => Busy()
+ case "shutting_down" => ShuttingDown()
+ case "error" => Error()
+ case "dead" => Dead()
+ case "success" => Success()
+ case _ => throw new IllegalArgumentException(s"Illegal session state: $s")
+ }
+ }
+
+ case class NotStarted() extends SessionState {
+ override def isActive: Boolean = true
+
+ override def toString: String = "not_started"
+ }
+
+ case class Starting() extends SessionState {
+ override def isActive: Boolean = true
+
+ override def toString: String = "starting"
+ }
+
+ case class Recovering() extends SessionState {
+ override def isActive: Boolean = true
+
+ override def toString: String = "recovering"
+ }
+
+ case class Idle() extends SessionState {
+ override def isActive: Boolean = true
+
+ override def toString: String = "idle"
+ }
+
+ case class Running() extends SessionState {
+ override def isActive: Boolean = true
+
+ override def toString: String = "running"
+ }
+
+ case class Busy() extends SessionState {
+ override def isActive: Boolean = true
+
+ override def toString: String = "busy"
+ }
+
+ case class ShuttingDown() extends SessionState {
+ override def isActive: Boolean = false
+
+ override def toString: String = "shutting_down"
+ }
+
+ case class Error(time: Long = System.nanoTime()) extends FinishedSessionState {
+ override def isActive: Boolean = true
+
+ override def toString: String = "error"
+ }
+
+ case class Dead(time: Long = System.nanoTime()) extends FinishedSessionState {
+ override def isActive: Boolean = false
+
+ override def toString: String = "dead"
+ }
+
+ case class Success(time: Long = System.nanoTime()) extends FinishedSessionState {
+ override def isActive: Boolean = false
+
+ override def toString: String = "success"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/test/scala/com/cloudera/livy/LivyBaseUnitTestSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/com/cloudera/livy/LivyBaseUnitTestSuite.scala b/core/src/test/scala/com/cloudera/livy/LivyBaseUnitTestSuite.scala
deleted file mode 100644
index d833e39..0000000
--- a/core/src/test/scala/com/cloudera/livy/LivyBaseUnitTestSuite.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy
-
-import org.scalatest.{Outcome, Suite}
-
-trait LivyBaseUnitTestSuite extends Suite with Logging {
-
- protected override def withFixture(test: NoArgTest): Outcome = {
- val testName = test.name
- val suiteName = this.getClass.getName
- try {
- info(s"\n\n==== TEST OUTPUT FOR $suiteName: '$testName' ====\n")
- test()
- } finally {
- info(s"\n\n==== FINISHED $suiteName: '$testName' ====\n")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/core/src/test/scala/org/apache/livy/LivyBaseUnitTestSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/livy/LivyBaseUnitTestSuite.scala b/core/src/test/scala/org/apache/livy/LivyBaseUnitTestSuite.scala
new file mode 100644
index 0000000..908172b
--- /dev/null
+++ b/core/src/test/scala/org/apache/livy/LivyBaseUnitTestSuite.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy
+
+import org.scalatest.{Outcome, Suite}
+
+trait LivyBaseUnitTestSuite extends Suite with Logging {
+
+ protected override def withFixture(test: NoArgTest): Outcome = {
+ val testName = test.name
+ val suiteName = this.getClass.getName
+ try {
+ info(s"\n\n==== TEST OUTPUT FOR $suiteName: '$testName' ====\n")
+ test()
+ } finally {
+ info(s"\n\n==== FINISHED $suiteName: '$testName' ====\n")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/coverage/pom.xml
----------------------------------------------------------------------
diff --git a/coverage/pom.xml b/coverage/pom.xml
index 2cf945a..de5ab2e 100644
--- a/coverage/pom.xml
+++ b/coverage/pom.xml
@@ -20,14 +20,14 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>livy-coverage-report</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index d1deec1..45bd11c 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,15 +21,15 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-examples</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -40,17 +40,17 @@
<dependencies>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-scala-api_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-client-http</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/examples/src/main/java/com/cloudera/livy/examples/PiApp.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/com/cloudera/livy/examples/PiApp.java b/examples/src/main/java/com/cloudera/livy/examples/PiApp.java
deleted file mode 100644
index 7971474..0000000
--- a/examples/src/main/java/com/cloudera/livy/examples/PiApp.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.examples;
-
-import java.io.File;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.api.java.function.Function2;
-
-import com.cloudera.livy.*;
-
-class PiJob implements
- Job<Double>,
- Function<Integer, Integer>,
- Function2<Integer, Integer, Integer> {
-
- private final int slices;
- private final int samples;
-
- public PiJob(int slices) {
- this.slices = slices;
- this.samples = (int) Math.min(100000L * slices, Integer.MAX_VALUE);
- }
-
- @Override
- public Double call(JobContext ctx) throws Exception {
- List<Integer> sampleList = new ArrayList<>();
- for (int i = 0; i < samples; i++) {
- sampleList.add(i);
- }
-
- return 4.0d * ctx.sc().parallelize(sampleList, slices).map(this).reduce(this) / samples;
- }
-
- @Override
- public Integer call(Integer v1) {
- double x = Math.random() * 2 - 1;
- double y = Math.random() * 2 - 1;
- return (x * x + y * y < 1) ? 1 : 0;
- }
-
- @Override
- public Integer call(Integer v1, Integer v2) {
- return v1 + v2;
- }
-}
-
-/**
- * Example execution:
- * java -cp /pathTo/spark-core_2.10-*version*.jar:/pathTo/livy-api-*version*.jar:
- * /pathTo/livy-client-http-*version*.jar:/pathTo/livy-examples-*version*.jar
- * com.cloudera.livy.examples.PiApp http://livy-host:8998 2
- */
-public class PiApp {
- public static void main(String[] args) throws Exception {
- if (args.length != 2) {
- System.err.println("Usage: PiJob <livy url> <slices>");
- System.exit(-1);
- }
-
- LivyClient client = new LivyClientBuilder()
- .setURI(new URI(args[0]))
- .build();
-
- try {
- System.out.println("Uploading livy-example jar to the SparkContext...");
- for (String s : System.getProperty("java.class.path").split(File.pathSeparator)) {
- if (new File(s).getName().startsWith("livy-examples")) {
- client.uploadJar(new File(s)).get();
- break;
- }
- }
-
- final int slices = Integer.parseInt(args[1]);
- double pi = client.submit(new PiJob(slices)).get();
-
- System.out.println("Pi is roughly " + pi);
- } finally {
- client.stop(true);
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/examples/src/main/java/org/apache/livy/examples/PiApp.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/livy/examples/PiApp.java b/examples/src/main/java/org/apache/livy/examples/PiApp.java
new file mode 100644
index 0000000..638f3b2
--- /dev/null
+++ b/examples/src/main/java/org/apache/livy/examples/PiApp.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.examples;
+
+import java.io.File;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.function.Function2;
+
+import org.apache.livy.*;
+
+class PiJob implements
+ Job<Double>,
+ Function<Integer, Integer>,
+ Function2<Integer, Integer, Integer> {
+
+ private final int slices;
+ private final int samples;
+
+ public PiJob(int slices) {
+ this.slices = slices;
+ this.samples = (int) Math.min(100000L * slices, Integer.MAX_VALUE);
+ }
+
+ @Override
+ public Double call(JobContext ctx) throws Exception {
+ List<Integer> sampleList = new ArrayList<>();
+ for (int i = 0; i < samples; i++) {
+ sampleList.add(i);
+ }
+
+ return 4.0d * ctx.sc().parallelize(sampleList, slices).map(this).reduce(this) / samples;
+ }
+
+ @Override
+ public Integer call(Integer v1) {
+ double x = Math.random() * 2 - 1;
+ double y = Math.random() * 2 - 1;
+ return (x * x + y * y < 1) ? 1 : 0;
+ }
+
+ @Override
+ public Integer call(Integer v1, Integer v2) {
+ return v1 + v2;
+ }
+}
+
+/**
+ * Example execution:
+ * java -cp /pathTo/spark-core_2.10-*version*.jar:/pathTo/livy-api-*version*.jar:
+ * /pathTo/livy-client-http-*version*.jar:/pathTo/livy-examples-*version*.jar
+ * org.apache.livy.examples.PiApp http://livy-host:8998 2
+ */
+public class PiApp {
+ public static void main(String[] args) throws Exception {
+ if (args.length != 2) {
+ System.err.println("Usage: PiJob <livy url> <slices>");
+ System.exit(-1);
+ }
+
+ LivyClient client = new LivyClientBuilder()
+ .setURI(new URI(args[0]))
+ .build();
+
+ try {
+ System.out.println("Uploading livy-example jar to the SparkContext...");
+ for (String s : System.getProperty("java.class.path").split(File.pathSeparator)) {
+ if (new File(s).getName().startsWith("livy-examples")) {
+ client.uploadJar(new File(s)).get();
+ break;
+ }
+ }
+
+ final int slices = Integer.parseInt(args[1]);
+ double pi = client.submit(new PiJob(slices)).get();
+
+ System.out.println("Pi is roughly " + pi);
+ } finally {
+ client.stop(true);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/examples/src/main/scala/com/cloudera/livy/examples/WordCountApp.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/com/cloudera/livy/examples/WordCountApp.scala b/examples/src/main/scala/com/cloudera/livy/examples/WordCountApp.scala
deleted file mode 100644
index 55b9da2..0000000
--- a/examples/src/main/scala/com/cloudera/livy/examples/WordCountApp.scala
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package com.cloudera.livy.examples
-
-import java.io.{File, FileNotFoundException}
-import java.net.URI
-
-import org.apache.spark.storage.StorageLevel
-import scala.concurrent.Await
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import com.cloudera.livy.LivyClientBuilder
-import com.cloudera.livy.scalaapi._
-
-/**
- * A WordCount example using Scala-API which reads text from a stream and saves
- * it as data frames. The word with maximum count is the result.
- */
-object WordCountApp {
-
- var scalaClient: LivyScalaClient = null
-
- /**
- * Initializes the Scala client with the given url.
- * @param url The Livy server url.
- */
- def init(url: String): Unit = {
- scalaClient = new LivyClientBuilder(false).setURI(new URI(url)).build().asScalaClient
- }
-
- /**
- * Uploads the Scala-API Jar and the examples Jar from the target directory.
- * @throws FileNotFoundException If either of Scala-API Jar or examples Jar is not found.
- */
- @throws(classOf[FileNotFoundException])
- def uploadRelevantJarsForJobExecution(): Unit = {
- val exampleAppJarPath = getSourcePath(this)
- val scalaApiJarPath = getSourcePath(scalaClient)
- uploadJar(exampleAppJarPath)
- uploadJar(scalaApiJarPath)
- }
-
- @throws(classOf[FileNotFoundException])
- private def getSourcePath(obj: Object): String = {
- val source = obj.getClass.getProtectionDomain.getCodeSource
- if (source != null && source.getLocation.getPath != "") {
- source.getLocation.getPath
- } else {
- throw new FileNotFoundException(s"Jar containing ${obj.getClass.getName} not found.")
- }
- }
-
- private def uploadJar(path: String) = {
- val file = new File(path)
- val uploadJarFuture = scalaClient.uploadJar(file)
- Await.result(uploadJarFuture, 40 second) match {
- case null => println("Successfully uploaded " + file.getName)
- }
- }
-
- /**
- * Submits a spark streaming job to the livy server.
- *
- * The streaming job reads data from the given host and port. The data read
- * is saved in json format as data frames in the given output path file. If the file is present
- * it appends to it, else creates a new file. For simplicity, the number of streaming batches
- * are 2 with each batch for 20 seconds. The Timeout of the streaming job is set to 40 seconds.
- * @param host Hostname that Spark Streaming context has to connect for receiving data.
- * @param port Port that Spark Streaming context has to connect for receiving data.
- * @param outputPath Output path to save the processed data read by the Spark Streaming context.
- */
- def processStreamingWordCount(
- host: String,
- port: Int,
- outputPath: String): ScalaJobHandle[Unit] = {
- scalaClient.submit { context =>
- context.createStreamingContext(15000)
- val ssc = context.streamingctx
- val sqlctx = context.sqlctx
- val lines = ssc.socketTextStream(host, port, StorageLevel.MEMORY_AND_DISK_SER)
- val words = lines.filter(filterEmptyContent(_)).flatMap(tokenize(_))
- words.print()
- words.foreachRDD { rdd =>
- import sqlctx.implicits._
- val df = rdd.toDF("word")
- df.write.mode("append").json(outputPath)
- }
- ssc.start()
- ssc.awaitTerminationOrTimeout(12000)
- ssc.stop(false, true)
- }
- }
-
- /**
- * Submits a spark sql job to the livy server.
- *
- * The sql context job reads data frames from the given json path and executes
- * a sql query to get the word with max count on the temp table created with data frames.
- * @param inputPath Input path to the json data containing the words.
- */
- def getWordWithMostCount(inputPath: String): ScalaJobHandle[String] = {
- scalaClient.submit { context =>
- val sqlctx = context.sqlctx
- val rdd = sqlctx.read.json(inputPath)
- rdd.registerTempTable("words")
- val result = sqlctx.sql("select word, count(word) as word_count from words " +
- "group by word order by word_count desc limit 1")
- result.first().toString()
- }
- }
-
- private def filterEmptyContent(text: String): Boolean = {
- text != null && !text.isEmpty
- }
-
- private def tokenize(text : String) : Array[String] = {
- text.toLowerCase.replaceAll("[^a-zA-Z0-9\\s]", "").split("\\s+")
- }
-
- private def stopClient(): Unit = {
- if (scalaClient != null) {
- scalaClient.stop(true)
- scalaClient = null;
- }
- }
-
- /**
- * Main method of the WordCount App. This method does the following
- * - Validate the arguments.
- * - Initializes the scala client of livy.
- * - Uploads the required livy and app code jar files to the spark cluster needed during runtime.
- * - Executes the streaming job that reads text-data from socket stream, tokenizes and saves
- * them as dataframes in JSON format in the given output path.
- * - Executes the sql-context job which reads the data frames from the given output path and
- * and returns the word with max count.
- *
- * @param args
- *
- * REQUIRED ARGUMENTS
- * arg(0) - Livy server url.
- * arg(1) - Output path to save the text read from the stream.
- *
- * Remaining arguments are treated as key=value pairs. The following keys are recognized:
- * host="examplehost" where "host" is the key and "examplehost" is the value
- * port=8080 where "port" is the key and 8080 is the value
- *
- * STEPS FOR EXECUTION - To get accurate results for one execution:
- * 1) Delete if the file already exists in the given outputFilePath.
- *
- * 2) Spark streaming will listen to the given or defaulted host and port. So textdata needs to be
- * passed as socket stream during the run-time of the App. The streaming context reads 2
- * batches of data with an interval of 20 seconds for each batch. All the data has to be
- * fed before the streaming context completes the second batch. NOTE - Inorder to get accurate
- * results for one execution, pass the textdata before the execution of the app so that all the
- * data is read by the socket stream.
- * To pass data to localhost and port 8086 provide the following command
- * nc -kl 8086
- *
- * 3) The text can be provided as paragraphs as the app will tokenize the data and filter spaces.
- *
- * 4) Execute the application jar file with the required and optional arguments either using
- * mvn or scala.
- *
- * Example execution:
- * scala -cp /pathTo/livy-api-*version*.jar:/pathTo/livy-client-http-*version*.jar:
- * /pathTo/livy-examples-*version*.jar:/pathTo/livy-scala-api-*version*.jar
- * com.cloudera.livy.examples.WordCountApp http://livy-host:8998 /outputFilePath
- * host=myhost port=8080
- */
- def main(args: Array[String]): Unit = {
- var socketStreamHost: String = "localhost"
- var socketStreamPort: Int = 8086
- var url = ""
- var outputFilePath = ""
- def parseOptionalArg(arg: String): Unit = {
- val Array(argKey, argValue) = arg.split("=")
- argKey match {
- case "host" => socketStreamHost = argValue
- case "port" => socketStreamPort = argValue.toInt
- case _ => throw new IllegalArgumentException("Invalid key for optional arguments")
- }
- }
- require(args.length >= 2 && args.length <= 4)
- url = args(0)
- outputFilePath = args(1)
- args.slice(2, args.length).foreach(parseOptionalArg)
- try {
- init(url)
- uploadRelevantJarsForJobExecution()
- println("Calling processStreamingWordCount")
- val handle1 = processStreamingWordCount(socketStreamHost, socketStreamPort, outputFilePath)
- Await.result(handle1, 100 second)
- println("Calling getWordWithMostCount")
- val handle = getWordWithMostCount(outputFilePath)
- println("Word with max count::" + Await.result(handle, 100 second))
- } finally {
- stopClient()
- }
- }
-}
-// scalastyle:off println
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/examples/src/main/scala/org/apache/livy/examples/WordCountApp.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/livy/examples/WordCountApp.scala b/examples/src/main/scala/org/apache/livy/examples/WordCountApp.scala
new file mode 100644
index 0000000..34c0da9
--- /dev/null
+++ b/examples/src/main/scala/org/apache/livy/examples/WordCountApp.scala
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// scalastyle:off println
+package org.apache.livy.examples
+
+import java.io.{File, FileNotFoundException}
+import java.net.URI
+
+import org.apache.spark.storage.StorageLevel
+import scala.concurrent.Await
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.apache.livy.LivyClientBuilder
+import org.apache.livy.scalaapi._
+
+/**
+ * A WordCount example using Scala-API which reads text from a stream and saves
+ * it as data frames. The word with maximum count is the result.
+ */
+object WordCountApp {
+
+ var scalaClient: LivyScalaClient = null
+
+ /**
+ * Initializes the Scala client with the given url.
+ * @param url The Livy server url.
+ */
+ def init(url: String): Unit = {
+ scalaClient = new LivyClientBuilder(false).setURI(new URI(url)).build().asScalaClient
+ }
+
+ /**
+ * Uploads the Scala-API Jar and the examples Jar from the target directory.
+ * @throws FileNotFoundException If either of Scala-API Jar or examples Jar is not found.
+ */
+ @throws(classOf[FileNotFoundException])
+ def uploadRelevantJarsForJobExecution(): Unit = {
+ val exampleAppJarPath = getSourcePath(this)
+ val scalaApiJarPath = getSourcePath(scalaClient)
+ uploadJar(exampleAppJarPath)
+ uploadJar(scalaApiJarPath)
+ }
+
+ @throws(classOf[FileNotFoundException])
+ private def getSourcePath(obj: Object): String = {
+ val source = obj.getClass.getProtectionDomain.getCodeSource
+ if (source != null && source.getLocation.getPath != "") {
+ source.getLocation.getPath
+ } else {
+ throw new FileNotFoundException(s"Jar containing ${obj.getClass.getName} not found.")
+ }
+ }
+
+ private def uploadJar(path: String) = {
+ val file = new File(path)
+ val uploadJarFuture = scalaClient.uploadJar(file)
+ Await.result(uploadJarFuture, 40 second) match {
+ case null => println("Successfully uploaded " + file.getName)
+ }
+ }
+
+ /**
+ * Submits a spark streaming job to the livy server.
+ *
+ * The streaming job reads data from the given host and port. The data read
+ * is saved in json format as data frames in the given output path file. If the file is present
+ * it appends to it, else creates a new file. For simplicity, the number of streaming batches
+ * are 2 with each batch for 20 seconds. The Timeout of the streaming job is set to 40 seconds.
+ * @param host Hostname that Spark Streaming context has to connect for receiving data.
+ * @param port Port that Spark Streaming context has to connect for receiving data.
+ * @param outputPath Output path to save the processed data read by the Spark Streaming context.
+ */
+ def processStreamingWordCount(
+ host: String,
+ port: Int,
+ outputPath: String): ScalaJobHandle[Unit] = {
+ scalaClient.submit { context =>
+ context.createStreamingContext(15000)
+ val ssc = context.streamingctx
+ val sqlctx = context.sqlctx
+ val lines = ssc.socketTextStream(host, port, StorageLevel.MEMORY_AND_DISK_SER)
+ val words = lines.filter(filterEmptyContent(_)).flatMap(tokenize(_))
+ words.print()
+ words.foreachRDD { rdd =>
+ import sqlctx.implicits._
+ val df = rdd.toDF("word")
+ df.write.mode("append").json(outputPath)
+ }
+ ssc.start()
+ ssc.awaitTerminationOrTimeout(12000)
+ ssc.stop(false, true)
+ }
+ }
+
+ /**
+ * Submits a spark sql job to the livy server.
+ *
+ * The sql context job reads data frames from the given json path and executes
+ * a sql query to get the word with max count on the temp table created with data frames.
+ * @param inputPath Input path to the json data containing the words.
+ */
+ def getWordWithMostCount(inputPath: String): ScalaJobHandle[String] = {
+ scalaClient.submit { context =>
+ val sqlctx = context.sqlctx
+ val rdd = sqlctx.read.json(inputPath)
+ rdd.registerTempTable("words")
+ val result = sqlctx.sql("select word, count(word) as word_count from words " +
+ "group by word order by word_count desc limit 1")
+ result.first().toString()
+ }
+ }
+
+ private def filterEmptyContent(text: String): Boolean = {
+ text != null && !text.isEmpty
+ }
+
+ private def tokenize(text : String) : Array[String] = {
+ text.toLowerCase.replaceAll("[^a-zA-Z0-9\\s]", "").split("\\s+")
+ }
+
+ private def stopClient(): Unit = {
+ if (scalaClient != null) {
+ scalaClient.stop(true)
+ scalaClient = null;
+ }
+ }
+
+ /**
+ * Main method of the WordCount App. This method does the following
+ * - Validate the arguments.
+ * - Initializes the scala client of livy.
+ * - Uploads the required livy and app code jar files to the spark cluster needed during runtime.
+ * - Executes the streaming job that reads text-data from socket stream, tokenizes and saves
+ * them as dataframes in JSON format in the given output path.
+ * - Executes the sql-context job which reads the data frames from the given output path and
+ * and returns the word with max count.
+ *
+ * @param args
+ *
+ * REQUIRED ARGUMENTS
+ * arg(0) - Livy server url.
+ * arg(1) - Output path to save the text read from the stream.
+ *
+ * Remaining arguments are treated as key=value pairs. The following keys are recognized:
+ * host="examplehost" where "host" is the key and "examplehost" is the value
+ * port=8080 where "port" is the key and 8080 is the value
+ *
+ * STEPS FOR EXECUTION - To get accurate results for one execution:
+ * 1) Delete if the file already exists in the given outputFilePath.
+ *
+ * 2) Spark streaming will listen to the given or defaulted host and port. So textdata needs to be
+ * passed as socket stream during the run-time of the App. The streaming context reads 2
+ * batches of data with an interval of 20 seconds for each batch. All the data has to be
+ * fed before the streaming context completes the second batch. NOTE - Inorder to get accurate
+ * results for one execution, pass the textdata before the execution of the app so that all the
+ * data is read by the socket stream.
+ * To pass data to localhost and port 8086 provide the following command
+ * nc -kl 8086
+ *
+ * 3) The text can be provided as paragraphs as the app will tokenize the data and filter spaces.
+ *
+ * 4) Execute the application jar file with the required and optional arguments either using
+ * mvn or scala.
+ *
+ * Example execution:
+ * scala -cp /pathTo/livy-api-*version*.jar:/pathTo/livy-client-http-*version*.jar:
+ * /pathTo/livy-examples-*version*.jar:/pathTo/livy-scala-api-*version*.jar
+ * com.cloudera.livy.examples.WordCountApp http://livy-host:8998 /outputFilePath
+ * host=myhost port=8080
+ */
+ def main(args: Array[String]): Unit = {
+ var socketStreamHost: String = "localhost"
+ var socketStreamPort: Int = 8086
+ var url = ""
+ var outputFilePath = ""
+ def parseOptionalArg(arg: String): Unit = {
+ val Array(argKey, argValue) = arg.split("=")
+ argKey match {
+ case "host" => socketStreamHost = argValue
+ case "port" => socketStreamPort = argValue.toInt
+ case _ => throw new IllegalArgumentException("Invalid key for optional arguments")
+ }
+ }
+ require(args.length >= 2 && args.length <= 4)
+ url = args(0)
+ outputFilePath = args(1)
+ args.slice(2, args.length).foreach(parseOptionalArg)
+ try {
+ init(url)
+ uploadRelevantJarsForJobExecution()
+ println("Calling processStreamingWordCount")
+ val handle1 = processStreamingWordCount(socketStreamHost, socketStreamPort, outputFilePath)
+ Await.result(handle1, 100 second)
+ println("Calling getWordWithMostCount")
+ val handle = getWordWithMostCount(outputFilePath)
+ println("Word with max count::" + Await.result(handle, 100 second))
+ } finally {
+ stopClient()
+ }
+ }
+}
+// scalastyle:off println
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/minicluster-dependencies/pom.xml
----------------------------------------------------------------------
diff --git a/integration-test/minicluster-dependencies/pom.xml b/integration-test/minicluster-dependencies/pom.xml
index d6a400c..809a3fc 100644
--- a/integration-test/minicluster-dependencies/pom.xml
+++ b/integration-test/minicluster-dependencies/pom.xml
@@ -23,13 +23,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>multi-scala-project-root</artifactId>
<relativePath>../../scala/pom.xml</relativePath>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>minicluster-dependencies-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<properties>
<skipDeploy>true</skipDeploy>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/minicluster-dependencies/scala-2.10/pom.xml
----------------------------------------------------------------------
diff --git a/integration-test/minicluster-dependencies/scala-2.10/pom.xml b/integration-test/minicluster-dependencies/scala-2.10/pom.xml
index b3493e9..12f6666 100644
--- a/integration-test/minicluster-dependencies/scala-2.10/pom.xml
+++ b/integration-test/minicluster-dependencies/scala-2.10/pom.xml
@@ -17,15 +17,15 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>minicluster-dependencies_2.10</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>minicluster-dependencies-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/minicluster-dependencies/scala-2.11/pom.xml
----------------------------------------------------------------------
diff --git a/integration-test/minicluster-dependencies/scala-2.11/pom.xml b/integration-test/minicluster-dependencies/scala-2.11/pom.xml
index d0e3fc7..d754593 100644
--- a/integration-test/minicluster-dependencies/scala-2.11/pom.xml
+++ b/integration-test/minicluster-dependencies/scala-2.11/pom.xml
@@ -17,15 +17,15 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>minicluster-dependencies_2.11</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>minicluster-dependencies-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
[09/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/LivyConf.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/LivyConf.scala b/server/src/main/scala/com/cloudera/livy/LivyConf.scala
deleted file mode 100644
index e4eb118..0000000
--- a/server/src/main/scala/com/cloudera/livy/LivyConf.scala
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy
-
-import java.io.File
-import java.lang.{Boolean => JBoolean, Long => JLong}
-import java.util.{Map => JMap}
-
-import scala.collection.JavaConverters._
-
-import org.apache.hadoop.conf.Configuration
-
-import com.cloudera.livy.client.common.ClientConf
-import com.cloudera.livy.client.common.ClientConf.ConfEntry
-import com.cloudera.livy.client.common.ClientConf.DeprecatedConf
-
-object LivyConf {
-
- case class Entry(override val key: String, override val dflt: AnyRef) extends ConfEntry
-
- object Entry {
- def apply(key: String, dflt: Boolean): Entry = Entry(key, dflt: JBoolean)
- def apply(key: String, dflt: Int): Entry = Entry(key, dflt: Integer)
- def apply(key: String, dflt: Long): Entry = Entry(key, dflt: JLong)
- }
-
- val TEST_MODE = ClientConf.TEST_MODE
-
- val SPARK_HOME = Entry("livy.server.spark-home", null)
- val LIVY_SPARK_MASTER = Entry("livy.spark.master", "local")
- val LIVY_SPARK_DEPLOY_MODE = Entry("livy.spark.deploy-mode", null)
-
- // Two configurations to specify Spark and related Scala version. These are internal
- // configurations will be set by LivyServer and used in session creation. It is not required to
- // set usually unless running with unofficial Spark + Scala versions
- // (like Spark 2.0 + Scala 2.10, Spark 1.6 + Scala 2.11)
- val LIVY_SPARK_SCALA_VERSION = Entry("livy.spark.scala-version", null)
- val LIVY_SPARK_VERSION = Entry("livy.spark.version", null)
-
- val SESSION_STAGING_DIR = Entry("livy.session.staging-dir", null)
- val FILE_UPLOAD_MAX_SIZE = Entry("livy.file.upload.max.size", 100L * 1024 * 1024)
- val LOCAL_FS_WHITELIST = Entry("livy.file.local-dir-whitelist", null)
- val ENABLE_HIVE_CONTEXT = Entry("livy.repl.enable-hive-context", false)
-
- val ENVIRONMENT = Entry("livy.environment", "production")
-
- val SERVER_HOST = Entry("livy.server.host", "0.0.0.0")
- val SERVER_PORT = Entry("livy.server.port", 8998)
-
- val UI_ENABLED = Entry("livy.ui.enabled", true)
-
- val REQUEST_HEADER_SIZE = Entry("livy.server.request-header.size", 131072)
- val RESPONSE_HEADER_SIZE = Entry("livy.server.response-header.size", 131072)
-
- val CSRF_PROTECTION = Entry("livy.server.csrf-protection.enabled", false)
-
- val IMPERSONATION_ENABLED = Entry("livy.impersonation.enabled", false)
- val SUPERUSERS = Entry("livy.superusers", null)
-
- val ACCESS_CONTROL_ENABLED = Entry("livy.server.access-control.enabled", false)
- val ACCESS_CONTROL_USERS = Entry("livy.server.access-control.users", null)
-
- val SSL_KEYSTORE = Entry("livy.keystore", null)
- val SSL_KEYSTORE_PASSWORD = Entry("livy.keystore.password", null)
- val SSL_KEY_PASSWORD = Entry("livy.key-password", null)
-
- val AUTH_TYPE = Entry("livy.server.auth.type", null)
- val AUTH_KERBEROS_PRINCIPAL = Entry("livy.server.auth.kerberos.principal", null)
- val AUTH_KERBEROS_KEYTAB = Entry("livy.server.auth.kerberos.keytab", null)
- val AUTH_KERBEROS_NAME_RULES = Entry("livy.server.auth.kerberos.name-rules", "DEFAULT")
-
- val HEARTBEAT_WATCHDOG_INTERVAL = Entry("livy.server.heartbeat-watchdog.interval", "1m")
-
- val LAUNCH_KERBEROS_PRINCIPAL = Entry("livy.server.launch.kerberos.principal", null)
- val LAUNCH_KERBEROS_KEYTAB = Entry("livy.server.launch.kerberos.keytab", null)
- val LAUNCH_KERBEROS_REFRESH_INTERVAL = Entry("livy.server.launch.kerberos.refresh-interval", "1h")
- val KINIT_FAIL_THRESHOLD = Entry("livy.server.launch.kerberos.kinit-fail-threshold", 5)
-
- /**
- * Recovery mode of Livy. Possible values:
- * off: Default. Turn off recovery. Every time Livy shuts down, it stops and forgets all sessions.
- * recovery: Livy persists session info to the state store. When Livy restarts, it recovers
- * previous sessions from the state store.
- * Must set livy.server.recovery.state-store and livy.server.recovery.state-store.url to
- * configure the state store.
- */
- val RECOVERY_MODE = Entry("livy.server.recovery.mode", "off")
- /**
- * Where Livy should store state to for recovery. Possible values:
- * <empty>: Default. State store disabled.
- * filesystem: Store state on a file system.
- * zookeeper: Store state in a Zookeeper instance.
- */
- val RECOVERY_STATE_STORE = Entry("livy.server.recovery.state-store", null)
- /**
- * For filesystem state store, the path of the state store directory. Please don't use a
- * filesystem that doesn't support atomic rename (e.g. S3). e.g. file:///tmp/livy or hdfs:///.
- * For zookeeper, the address to the Zookeeper servers. e.g. host1:port1,host2:port2
- */
- val RECOVERY_STATE_STORE_URL = Entry("livy.server.recovery.state-store.url", "")
-
- // If Livy can't find the yarn app within this time, consider it lost.
- val YARN_APP_LOOKUP_TIMEOUT = Entry("livy.server.yarn.app-lookup-timeout", "60s")
-
- // How often Livy polls YARN to refresh YARN app state.
- val YARN_POLL_INTERVAL = Entry("livy.server.yarn.poll-interval", "5s")
-
- // Days to keep Livy server request logs.
- val REQUEST_LOG_RETAIN_DAYS = Entry("livy.server.request-log-retain.days", 5)
-
- // REPL related jars separated with comma.
- val REPL_JARS = Entry("livy.repl.jars", null)
- // RSC related jars separated with comma.
- val RSC_JARS = Entry("livy.rsc.jars", null)
-
- // How long to check livy session leakage
- val YARN_APP_LEAKAGE_CHECK_TIMEOUT = Entry("livy.server.yarn.app-leakage.check-timeout", "600s")
- // how often to check livy session leakage
- val YARN_APP_LEAKAGE_CHECK_INTERVAL = Entry("livy.server.yarn.app-leakage.check-interval", "60s")
-
- // Whether session timeout should be checked, by default it will be checked, which means inactive
- // session will be stopped after "livy.server.session.timeout"
- val SESSION_TIMEOUT_CHECK = Entry("livy.server.session.timeout-check", true)
- // How long will an inactive session be gc-ed.
- val SESSION_TIMEOUT = Entry("livy.server.session.timeout", "1h")
- // How long a finished session state will be kept in memory
- val SESSION_STATE_RETAIN_TIME = Entry("livy.server.session.state-retain.sec", "600s")
-
- val SPARK_MASTER = "spark.master"
- val SPARK_DEPLOY_MODE = "spark.submit.deployMode"
- val SPARK_JARS = "spark.jars"
- val SPARK_FILES = "spark.files"
- val SPARK_ARCHIVES = "spark.yarn.dist.archives"
- val SPARK_PY_FILES = "spark.submit.pyFiles"
-
- /**
- * These are Spark configurations that contain lists of files that the user can add to
- * their jobs in one way or another. Livy needs to pre-process these to make sure the
- * user can read them (in case they reference local files), and to provide correct URIs
- * to Spark based on the Livy config.
- *
- * The configuration allows adding new configurations in case we either forget something in
- * the hardcoded list, or new versions of Spark add new configs.
- */
- val SPARK_FILE_LISTS = Entry("livy.spark.file-list-configs", null)
-
- private val HARDCODED_SPARK_FILE_LISTS = Seq(
- SPARK_JARS,
- SPARK_FILES,
- SPARK_ARCHIVES,
- SPARK_PY_FILES,
- "spark.yarn.archive",
- "spark.yarn.dist.files",
- "spark.yarn.dist.jars",
- "spark.yarn.jar",
- "spark.yarn.jars"
- )
-
- case class DepConf(
- override val key: String,
- override val version: String,
- override val deprecationMessage: String = "")
- extends DeprecatedConf
-
- private val configsWithAlternatives: Map[String, DeprecatedConf] = Map[String, DepConf](
- LIVY_SPARK_DEPLOY_MODE.key -> DepConf("livy.spark.deployMode", "0.4"),
- LIVY_SPARK_SCALA_VERSION.key -> DepConf("livy.spark.scalaVersion", "0.4"),
- ENABLE_HIVE_CONTEXT.key -> DepConf("livy.repl.enableHiveContext", "0.4"),
- CSRF_PROTECTION.key -> DepConf("livy.server.csrf_protection.enabled", "0.4"),
- ACCESS_CONTROL_ENABLED.key -> DepConf("livy.server.access_control.enabled", "0.4"),
- ACCESS_CONTROL_USERS.key -> DepConf("livy.server.access_control.users", "0.4"),
- AUTH_KERBEROS_NAME_RULES.key -> DepConf("livy.server.auth.kerberos.name_rules", "0.4"),
- LAUNCH_KERBEROS_REFRESH_INTERVAL.key ->
- DepConf("livy.server.launch.kerberos.refresh_interval", "0.4"),
- KINIT_FAIL_THRESHOLD.key -> DepConf("livy.server.launch.kerberos.kinit_fail_threshold", "0.4"),
- YARN_APP_LEAKAGE_CHECK_TIMEOUT.key ->
- DepConf("livy.server.yarn.app-leakage.check_timeout", "0.4"),
- YARN_APP_LEAKAGE_CHECK_INTERVAL.key ->
- DepConf("livy.server.yarn.app-leakage.check_interval", "0.4")
- )
-
- private val deprecatedConfigs: Map[String, DeprecatedConf] = {
- val configs: Seq[DepConf] = Seq(
- // There are no deprecated configs without alternatives currently.
- )
-
- Map(configs.map { cfg => (cfg.key -> cfg) }: _*)
- }
-
-}
-
-/**
- *
- * @param loadDefaults whether to also load values from the Java system properties
- */
-class LivyConf(loadDefaults: Boolean) extends ClientConf[LivyConf](null) {
-
- import LivyConf._
-
- private lazy val _superusers = configToSeq(SUPERUSERS)
- private lazy val _allowedUsers = configToSeq(ACCESS_CONTROL_USERS).toSet
-
- lazy val hadoopConf = new Configuration()
- lazy val localFsWhitelist = configToSeq(LOCAL_FS_WHITELIST).map { path =>
- // Make sure the path ends with a single separator.
- path.stripSuffix("/") + "/"
- }
-
- lazy val sparkFileLists = HARDCODED_SPARK_FILE_LISTS ++ configToSeq(SPARK_FILE_LISTS)
-
- /**
- * Create a LivyConf that loads defaults from the system properties and the classpath.
- * @return
- */
- def this() = this(true)
-
- if (loadDefaults) {
- loadFromMap(sys.props)
- }
-
- def loadFromFile(name: String): LivyConf = {
- getConfigFile(name)
- .map(Utils.getPropertiesFromFile)
- .foreach(loadFromMap)
- this
- }
-
- /** Return true if spark master starts with yarn. */
- def isRunningOnYarn(): Boolean = sparkMaster().startsWith("yarn")
-
- /** Return the spark deploy mode Livy sessions should use. */
- def sparkDeployMode(): Option[String] = Option(get(LIVY_SPARK_DEPLOY_MODE)).filterNot(_.isEmpty)
-
- /** Return the location of the spark home directory */
- def sparkHome(): Option[String] = Option(get(SPARK_HOME)).orElse(sys.env.get("SPARK_HOME"))
-
- /** Return the spark master Livy sessions should use. */
- def sparkMaster(): String = get(LIVY_SPARK_MASTER)
-
- /** Return the path to the spark-submit executable. */
- def sparkSubmit(): String = {
- sparkHome().map { _ + File.separator + "bin" + File.separator + "spark-submit" }.get
- }
-
- /** Return the list of superusers. */
- def superusers(): Seq[String] = _superusers
-
- /** Return the set of users allowed to use Livy via SPNEGO. */
- def allowedUsers(): Set[String] = _allowedUsers
-
- private val configDir: Option[File] = {
- sys.env.get("LIVY_CONF_DIR")
- .orElse(sys.env.get("LIVY_HOME").map(path => s"$path${File.separator}conf"))
- .map(new File(_))
- .filter(_.exists())
- }
-
- private def getConfigFile(name: String): Option[File] = {
- configDir.map(new File(_, name)).filter(_.exists())
- }
-
- private def loadFromMap(map: Iterable[(String, String)]): Unit = {
- map.foreach { case (k, v) =>
- if (k.startsWith("livy.")) {
- set(k, v)
- }
- }
- }
-
- private def configToSeq(entry: LivyConf.Entry): Seq[String] = {
- Option(get(entry)).map(_.split("[, ]+").toSeq).getOrElse(Nil)
- }
-
- override def getConfigsWithAlternatives: JMap[String, DeprecatedConf] = {
- configsWithAlternatives.asJava
- }
-
- override def getDeprecatedConfigs: JMap[String, DeprecatedConf] = {
- deprecatedConfigs.asJava
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/package.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/package.scala b/server/src/main/scala/com/cloudera/livy/package.scala
deleted file mode 100644
index c1661a9..0000000
--- a/server/src/main/scala/com/cloudera/livy/package.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera
-
-import java.util.Properties
-
-import scala.util.control.NonFatal
-
-package object livy {
-
- private object LivyBuildInfo {
- val (
- livyVersion: String,
- livyBuildUser: String,
- livyRevision: String,
- livyBranch: String,
- livyBuildDate: String,
- livyRepo: String
- ) = {
- val unknown = "<unknown>"
- val defaultValue = (unknown, unknown, unknown, unknown, unknown, unknown)
- val resource = Option(Thread.currentThread().getContextClassLoader
- .getResourceAsStream("livy-version-info.properties"))
-
- try {
- resource.map { r =>
- val properties = new Properties()
- properties.load(r)
- (
- properties.getProperty("version", unknown),
- properties.getProperty("user", unknown),
- properties.getProperty("revision", unknown),
- properties.getProperty("branch", unknown),
- properties.getProperty("date", unknown),
- properties.getProperty("url", unknown)
- )
- }.getOrElse(defaultValue)
- } catch {
- case NonFatal(e) =>
- // swallow the exception
- defaultValue
- } finally {
- try {
- resource.foreach(_.close())
- } catch {
- case NonFatal(e) => // swallow the exception in closing the stream
- }
- }
- }
- }
-
- val LIVY_VERSION = LivyBuildInfo.livyVersion
- val LIVY_BUILD_USER = LivyBuildInfo.livyBuildUser
- val LIVY_REVISION = LivyBuildInfo.livyRevision
- val LIVY_BRANCH = LivyBuildInfo.livyBranch
- val LIVY_BUILD_DATE = LivyBuildInfo.livyBuildDate
- val LIVY_REPO_URL = LivyBuildInfo.livyRepo
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/AccessFilter.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/AccessFilter.scala b/server/src/main/scala/com/cloudera/livy/server/AccessFilter.scala
deleted file mode 100644
index 1138cc6..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/AccessFilter.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-import javax.servlet._
-import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
-
-import com.cloudera.livy.LivyConf
-
-class AccessFilter(livyConf: LivyConf) extends Filter {
-
- override def init(filterConfig: FilterConfig): Unit = {}
-
- override def doFilter(request: ServletRequest,
- response: ServletResponse,
- chain: FilterChain): Unit = {
- val httpRequest = request.asInstanceOf[HttpServletRequest]
- val remoteUser = httpRequest.getRemoteUser
- if (livyConf.allowedUsers.contains(remoteUser)) {
- chain.doFilter(request, response)
- } else {
- val httpServletResponse = response.asInstanceOf[HttpServletResponse]
- httpServletResponse.sendError(HttpServletResponse.SC_UNAUTHORIZED,
- "User not authorised to use Livy.")
- }
- }
-
- override def destroy(): Unit = {}
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/ApiVersioningSupport.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/ApiVersioningSupport.scala b/server/src/main/scala/com/cloudera/livy/server/ApiVersioningSupport.scala
deleted file mode 100644
index ff2f05f..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/ApiVersioningSupport.scala
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-import javax.servlet.http.HttpServletRequest
-
-import org.scalatra.{NotAcceptable, ScalatraBase}
-
-/**
- * Livy's servlets can mix-in this trait to get API version support.
- *
- * Example: {{{
- * import ApiVersions._
- * class FooServlet
- * ...
- * with ApiVersioningSupport
- * ...
- * {
- * get("/test") {
- * ...
- * }
- * get("/test", apiVersion <= v0_2) {
- * ...
- * }
- * get("/test", apiVersion <= v0_1) {
- * ...
- * }
- * }
- * }}}
- */
-trait ApiVersioningSupport extends AbstractApiVersioningSupport {
- this: ScalatraBase =>
- // Link the abstract trait to Livy's version enum.
- override val apiVersions = ApiVersions
- override type ApiVersionType = ApiVersions.Value
-}
-
-trait AbstractApiVersioningSupport {
- this: ScalatraBase =>
- protected val apiVersions: Enumeration
- protected type ApiVersionType
-
- /**
- * Before proceeding with routing, validate the specified API version in the request.
- * If validation passes, cache the parsed API version as a per-request attribute.
- */
- before() {
- request(AbstractApiVersioningSupport.ApiVersionKey) = request.getHeader("Accept") match {
- case acceptHeader @ AbstractApiVersioningSupport.AcceptHeaderRegex(apiVersion) =>
- try {
- apiVersions.withName(apiVersion).asInstanceOf[ApiVersionType]
- } catch {
- case e: NoSuchElementException =>
- halt(NotAcceptable(e.getMessage))
- }
- case _ =>
- // Return the latest version.
- apiVersions.apply(apiVersions.maxId - 1).asInstanceOf[ApiVersionType]
- }
- }
-
- /**
- * @return The specified API version in the request.
- */
- def apiVersion: ApiVersionType = {
- request(AbstractApiVersioningSupport.ApiVersionKey).asInstanceOf[ApiVersionType]
- }
-
-}
-
-object AbstractApiVersioningSupport {
- // Get every character after "application/vnd.livy.v" until hitting a + sign.
- private final val AcceptHeaderRegex = """application/vnd\.livy\.v([^\+]*).*""".r
-
- // AbstractApiVersioningSupport uses a per-request attribute to store the parsed API version.
- // This is the key name for the attribute.
- private final val ApiVersionKey = "apiVersion"
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/ApiVersions.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/ApiVersions.scala b/server/src/main/scala/com/cloudera/livy/server/ApiVersions.scala
deleted file mode 100644
index 34275a4..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/ApiVersions.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-/**
- * This enum defines Livy's API versions.
- * [[com.cloudera.livy.server.AbstractApiVersioningSupport]] uses this for API version checking.
- *
- * Version is defined as <major version>.<minor version>.
- * When making backward compatible change (adding methods/fields), bump minor version.
- * When making backward incompatible change (renaming/removing methods/fields), bump major version.
- * This ensures our users can safely migrate to a newer API version if major version is unchanged.
- */
-object ApiVersions extends Enumeration {
- type ApiVersions = Value
- // ApiVersions are ordered and the ordering is defined by the order of Value() calls.
- // Please make sure API version is defined in ascending order (Older API before newer).
- // AbstractApiVersioningSupport relies on the ordering.
- val v0_1 = Value("0.1")
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/CsrfFilter.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/CsrfFilter.scala b/server/src/main/scala/com/cloudera/livy/server/CsrfFilter.scala
deleted file mode 100644
index d2aa011..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/CsrfFilter.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package com.cloudera.livy.server
-
-import javax.servlet._
-import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
-
-
-class CsrfFilter extends Filter {
-
- val METHODS_TO_IGNORE = Set("GET", "OPTIONS", "HEAD");
-
- val HEADER_NAME = "X-Requested-By";
-
- override def init(filterConfig: FilterConfig): Unit = {}
-
- override def doFilter(request: ServletRequest,
- response: ServletResponse,
- chain: FilterChain): Unit = {
- val httpRequest = request.asInstanceOf[HttpServletRequest]
-
- if (!METHODS_TO_IGNORE.contains(httpRequest.getMethod)
- && httpRequest.getHeader(HEADER_NAME) == null) {
- response.asInstanceOf[HttpServletResponse].sendError(HttpServletResponse.SC_BAD_REQUEST,
- "Missing Required Header for CSRF protection.")
- } else {
- chain.doFilter(request, response)
- }
- }
-
- override def destroy(): Unit = {}
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/JsonServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/JsonServlet.scala b/server/src/main/scala/com/cloudera/livy/server/JsonServlet.scala
deleted file mode 100644
index 7af6cce..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/JsonServlet.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
-
-import scala.concurrent.ExecutionContext
-import scala.reflect.ClassTag
-
-import com.fasterxml.jackson.core.JsonParseException
-import com.fasterxml.jackson.databind.{JsonMappingException, ObjectMapper}
-import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
-import org.scalatra._
-
-/**
- * An abstract servlet that provides overridden implementations for "post", "put" and "patch"
- * that can deserialize JSON data directly into user-defined types, without having to go through
- * a json4s intermediate. Results are also automatically serialized into JSON if the content type
- * says so.
- *
- * Serialization and deserialization are done through Jackson directly, so all Jackson features
- * are available.
- */
-abstract class JsonServlet extends ScalatraServlet with ApiFormats with FutureSupport {
-
- override protected implicit def executor: ExecutionContext = ExecutionContext.global
-
- private lazy val _defaultMapper = new ObjectMapper()
- .registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
-
- /**
- * Override this method if you need a custom Jackson object mapper; the default mapper
- * has the default configuration, plus the Scala module.
- */
- protected def createMapper(): ObjectMapper = _defaultMapper
-
- protected final val mapper = createMapper()
-
- before() {
- contentType = formats("json")
- }
-
- error {
- case e: JsonParseException => BadRequest(e.getMessage)
- case e: UnrecognizedPropertyException => BadRequest(e.getMessage)
- case e: JsonMappingException => BadRequest(e.getMessage)
- case e =>
- SessionServlet.error("internal error", e)
- InternalServerError(e.toString)
- }
-
- protected def jpatch[T: ClassTag](t: RouteTransformer*)(action: T => Any): Route = {
- patch(t: _*) {
- doAction(request, action)
- }
- }
-
- protected def jpost[T: ClassTag](t: RouteTransformer*)(action: T => Any): Route = {
- post(t: _*) {
- doAction(request, action)
- }
- }
-
- protected def jput[T: ClassTag](t: RouteTransformer*)(action: T => Any): Route = {
- put(t: _*) {
- doAction(request, action)
- }
- }
-
- override protected def renderResponseBody(actionResult: Any): Unit = {
- val result = actionResult match {
- case ActionResult(status, body, headers) if format == "json" =>
- ActionResult(status, toJson(body), headers)
- case str: String if format == "json" =>
- // This should be changed when we implement LIVY-54. For now, just create a dummy
- // JSON object when a raw string is being returned.
- toJson(Map("msg" -> str))
- case other if format == "json" =>
- toJson(other)
- case other =>
- other
- }
- super.renderResponseBody(result)
- }
-
- protected def bodyAs[T: ClassTag](req: HttpServletRequest)
- (implicit klass: ClassTag[T]): T = {
- bodyAs(req, klass.runtimeClass)
- }
-
- private def bodyAs[T](req: HttpServletRequest, klass: Class[_]): T = {
- mapper.readValue(req.getInputStream(), klass).asInstanceOf[T]
- }
-
- private def doAction[T: ClassTag](
- req: HttpServletRequest,
- action: T => Any)(implicit klass: ClassTag[T]): Any = {
- action(bodyAs[T](req, klass.runtimeClass))
- }
-
- private def isJson(res: HttpServletResponse, headers: Map[String, String] = Map()): Boolean = {
- val ctypeHeader = "Content-Type"
- headers.get(ctypeHeader).orElse(Option(res.getHeader(ctypeHeader)))
- .map(_.startsWith("application/json")).getOrElse(false)
- }
-
- private def toResult(obj: Any, res: HttpServletResponse): Any = obj match {
- case async: AsyncResult =>
- new AsyncResult {
- val is = async.is.map(toResult(_, res))
- }
- case ActionResult(status, body, headers) if isJson(res, headers) =>
- ActionResult(status, toJson(body), headers)
- case body if isJson(res) =>
- Ok(toJson(body))
- case other =>
- other
- }
-
- private def toJson(obj: Any): Any = {
- if (obj != null && obj != ()) {
- mapper.writeValueAsBytes(obj)
- } else {
- null
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/LivyServer.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/LivyServer.scala b/server/src/main/scala/com/cloudera/livy/server/LivyServer.scala
deleted file mode 100644
index 54f00ca..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/LivyServer.scala
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-import java.io.{BufferedInputStream, InputStream}
-import java.util.concurrent._
-import java.util.EnumSet
-import javax.servlet._
-
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.Future
-
-import org.apache.hadoop.security.{SecurityUtil, UserGroupInformation}
-import org.apache.hadoop.security.authentication.server._
-import org.eclipse.jetty.servlet.FilterHolder
-import org.scalatra.{NotFound, ScalatraServlet}
-import org.scalatra.metrics.MetricsBootstrap
-import org.scalatra.metrics.MetricsSupportExtensions._
-import org.scalatra.servlet.{MultipartConfig, ServletApiImplicits}
-
-import com.cloudera.livy._
-import com.cloudera.livy.server.batch.BatchSessionServlet
-import com.cloudera.livy.server.interactive.InteractiveSessionServlet
-import com.cloudera.livy.server.recovery.{SessionStore, StateStore}
-import com.cloudera.livy.server.ui.UIServlet
-import com.cloudera.livy.sessions.{BatchSessionManager, InteractiveSessionManager}
-import com.cloudera.livy.sessions.SessionManager.SESSION_RECOVERY_MODE_OFF
-import com.cloudera.livy.utils.LivySparkUtils._
-import com.cloudera.livy.utils.SparkYarnApp
-
-class LivyServer extends Logging {
-
- import LivyConf._
-
- private var server: WebServer = _
- private var _serverUrl: Option[String] = None
- // make livyConf accessible for testing
- private[livy] var livyConf: LivyConf = _
-
- private var kinitFailCount: Int = 0
- private var executor: ScheduledExecutorService = _
-
- def start(): Unit = {
- livyConf = new LivyConf().loadFromFile("livy.conf")
-
- val host = livyConf.get(SERVER_HOST)
- val port = livyConf.getInt(SERVER_PORT)
- val multipartConfig = MultipartConfig(
- maxFileSize = Some(livyConf.getLong(LivyConf.FILE_UPLOAD_MAX_SIZE))
- ).toMultipartConfigElement
-
- // Make sure the `spark-submit` program exists, otherwise much of livy won't work.
- testSparkHome(livyConf)
-
- // Test spark-submit and get Spark Scala version accordingly.
- val (sparkVersion, scalaVersionFromSparkSubmit) = sparkSubmitVersion(livyConf)
- testSparkVersion(sparkVersion)
-
- // If Spark and Scala version is set manually, should verify if they're consistent with
- // ones parsed from "spark-submit --version"
- val formattedSparkVersion = formatSparkVersion(sparkVersion)
- Option(livyConf.get(LIVY_SPARK_VERSION)).map(formatSparkVersion).foreach { version =>
- require(formattedSparkVersion == version,
- s"Configured Spark version $version is not equal to Spark version $formattedSparkVersion " +
- "got from spark-submit -version")
- }
-
- // Set formatted Spark and Scala version into livy configuration, this will be used by
- // session creation.
- // TODO Create a new class to pass variables from LivyServer to sessions and remove these
- // internal LivyConfs.
- livyConf.set(LIVY_SPARK_VERSION.key, formattedSparkVersion.productIterator.mkString("."))
- livyConf.set(LIVY_SPARK_SCALA_VERSION.key,
- sparkScalaVersion(formattedSparkVersion, scalaVersionFromSparkSubmit, livyConf))
-
- if (UserGroupInformation.isSecurityEnabled) {
- // If Hadoop security is enabled, run kinit periodically. runKinit() should be called
- // before any Hadoop operation, otherwise Kerberos exception will be thrown.
- executor = Executors.newScheduledThreadPool(1,
- new ThreadFactory() {
- override def newThread(r: Runnable): Thread = {
- val thread = new Thread(r)
- thread.setName("kinit-thread")
- thread.setDaemon(true)
- thread
- }
- }
- )
- val launch_keytab = livyConf.get(LAUNCH_KERBEROS_KEYTAB)
- val launch_principal = SecurityUtil.getServerPrincipal(
- livyConf.get(LAUNCH_KERBEROS_PRINCIPAL), host)
- require(launch_keytab != null,
- s"Kerberos requires ${LAUNCH_KERBEROS_KEYTAB.key} to be provided.")
- require(launch_principal != null,
- s"Kerberos requires ${LAUNCH_KERBEROS_PRINCIPAL.key} to be provided.")
- if (!runKinit(launch_keytab, launch_principal)) {
- error("Failed to run kinit, stopping the server.")
- sys.exit(1)
- }
- startKinitThread(launch_keytab, launch_principal)
- }
-
- testRecovery(livyConf)
-
- // Initialize YarnClient ASAP to save time.
- if (livyConf.isRunningOnYarn()) {
- SparkYarnApp.init(livyConf)
- Future { SparkYarnApp.yarnClient }
- }
-
- StateStore.init(livyConf)
- val sessionStore = new SessionStore(livyConf)
- val batchSessionManager = new BatchSessionManager(livyConf, sessionStore)
- val interactiveSessionManager = new InteractiveSessionManager(livyConf, sessionStore)
-
- server = new WebServer(livyConf, host, port)
- server.context.setResourceBase("src/main/com/cloudera/livy/server")
-
- val livyVersionServlet = new JsonServlet {
- before() { contentType = "application/json" }
-
- get("/") {
- Map("version" -> LIVY_VERSION,
- "user" -> LIVY_BUILD_USER,
- "revision" -> LIVY_REVISION,
- "branch" -> LIVY_BRANCH,
- "date" -> LIVY_BUILD_DATE,
- "url" -> LIVY_REPO_URL)
- }
- }
-
- // Servlet for hosting static files such as html, css, and js
- // Necessary since Jetty cannot set it's resource base inside a jar
- // Returns 404 if the file does not exist
- val staticResourceServlet = new ScalatraServlet {
- get("/*") {
- val fileName = params("splat")
- val notFoundMsg = "File not found"
-
- if (!fileName.isEmpty) {
- getClass.getResourceAsStream(s"ui/static/$fileName") match {
- case is: InputStream => new BufferedInputStream(is)
- case null => NotFound(notFoundMsg)
- }
- } else {
- NotFound(notFoundMsg)
- }
- }
- }
-
- def uiRedirectServlet(path: String) = new ScalatraServlet {
- get("/") {
- redirect(path)
- }
- }
-
- server.context.addEventListener(
- new ServletContextListener() with MetricsBootstrap with ServletApiImplicits {
-
- private def mount(sc: ServletContext, servlet: Servlet, mappings: String*): Unit = {
- val registration = sc.addServlet(servlet.getClass().getName(), servlet)
- registration.addMapping(mappings: _*)
- registration.setMultipartConfig(multipartConfig)
- }
-
- override def contextDestroyed(sce: ServletContextEvent): Unit = {
-
- }
-
- override def contextInitialized(sce: ServletContextEvent): Unit = {
- try {
- val context = sce.getServletContext()
- context.initParameters(org.scalatra.EnvironmentKey) = livyConf.get(ENVIRONMENT)
-
- val interactiveServlet =
- new InteractiveSessionServlet(interactiveSessionManager, sessionStore, livyConf)
- mount(context, interactiveServlet, "/sessions/*")
-
- val batchServlet = new BatchSessionServlet(batchSessionManager, sessionStore, livyConf)
- mount(context, batchServlet, "/batches/*")
-
- if (livyConf.getBoolean(UI_ENABLED)) {
- val uiServlet = new UIServlet
- mount(context, uiServlet, "/ui/*")
- mount(context, staticResourceServlet, "/static/*")
- mount(context, uiRedirectServlet("/ui/"), "/*")
- } else {
- mount(context, uiRedirectServlet("/metrics"), "/*")
- }
-
- context.mountMetricsAdminServlet("/metrics")
-
- mount(context, livyVersionServlet, "/version/*")
- } catch {
- case e: Throwable =>
- error("Exception thrown when initializing server", e)
- sys.exit(1)
- }
- }
-
- })
-
- livyConf.get(AUTH_TYPE) match {
- case authType @ KerberosAuthenticationHandler.TYPE =>
- val principal = SecurityUtil.getServerPrincipal(livyConf.get(AUTH_KERBEROS_PRINCIPAL),
- server.host)
- val keytab = livyConf.get(AUTH_KERBEROS_KEYTAB)
- require(principal != null,
- s"Kerberos auth requires ${AUTH_KERBEROS_PRINCIPAL.key} to be provided.")
- require(keytab != null,
- s"Kerberos auth requires ${AUTH_KERBEROS_KEYTAB.key} to be provided.")
-
- val holder = new FilterHolder(new AuthenticationFilter())
- holder.setInitParameter(AuthenticationFilter.AUTH_TYPE, authType)
- holder.setInitParameter(KerberosAuthenticationHandler.PRINCIPAL, principal)
- holder.setInitParameter(KerberosAuthenticationHandler.KEYTAB, keytab)
- holder.setInitParameter(KerberosAuthenticationHandler.NAME_RULES,
- livyConf.get(AUTH_KERBEROS_NAME_RULES))
- server.context.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType]))
- info(s"SPNEGO auth enabled (principal = $principal)")
-
- case null =>
- // Nothing to do.
-
- case other =>
- throw new IllegalArgumentException(s"Invalid auth type: $other")
- }
-
- if (livyConf.getBoolean(CSRF_PROTECTION)) {
- info("CSRF protection is enabled.")
- val csrfHolder = new FilterHolder(new CsrfFilter())
- server.context.addFilter(csrfHolder, "/*", EnumSet.allOf(classOf[DispatcherType]))
- }
-
- if (livyConf.getBoolean(ACCESS_CONTROL_ENABLED)) {
- if (livyConf.get(AUTH_TYPE) != null) {
- info("Access control is enabled.")
- val accessHolder = new FilterHolder(new AccessFilter(livyConf))
- server.context.addFilter(accessHolder, "/*", EnumSet.allOf(classOf[DispatcherType]))
- } else {
- throw new IllegalArgumentException("Access control was requested but could " +
- "not be enabled, since authentication is disabled.")
- }
- }
-
- server.start()
-
- Runtime.getRuntime().addShutdownHook(new Thread("Livy Server Shutdown") {
- override def run(): Unit = {
- info("Shutting down Livy server.")
- server.stop()
- }
- })
-
- _serverUrl = Some(s"${server.protocol}://${server.host}:${server.port}")
- sys.props("livy.server.server-url") = _serverUrl.get
- }
-
- def runKinit(keytab: String, principal: String): Boolean = {
- val commands = Seq("kinit", "-kt", keytab, principal)
- val proc = new ProcessBuilder(commands: _*).inheritIO().start()
- proc.waitFor() match {
- case 0 =>
- debug("Ran kinit command successfully.")
- kinitFailCount = 0
- true
- case _ =>
- warn("Fail to run kinit command.")
- kinitFailCount += 1
- false
- }
- }
-
- def startKinitThread(keytab: String, principal: String): Unit = {
- val refreshInterval = livyConf.getTimeAsMs(LAUNCH_KERBEROS_REFRESH_INTERVAL)
- val kinitFailThreshold = livyConf.getInt(KINIT_FAIL_THRESHOLD)
- executor.schedule(
- new Runnable() {
- override def run(): Unit = {
- if (runKinit(keytab, principal)) {
- // schedule another kinit run with a fixed delay.
- executor.schedule(this, refreshInterval, TimeUnit.MILLISECONDS)
- } else {
- // schedule another retry at once or fail the livy server if too many times kinit fail
- if (kinitFailCount >= kinitFailThreshold) {
- error(s"Exit LivyServer after ${kinitFailThreshold} times failures running kinit.")
- if (server.server.isStarted()) {
- stop()
- } else {
- sys.exit(1)
- }
- } else {
- executor.submit(this)
- }
- }
- }
- }, refreshInterval, TimeUnit.MILLISECONDS)
- }
-
- def join(): Unit = server.join()
-
- def stop(): Unit = {
- if (server != null) {
- server.stop()
- }
- }
-
- def serverUrl(): String = {
- _serverUrl.getOrElse(throw new IllegalStateException("Server not yet started."))
- }
-
- private[livy] def testRecovery(livyConf: LivyConf): Unit = {
- if (!livyConf.isRunningOnYarn()) {
- // If recovery is turned on but we are not running on YARN, quit.
- require(livyConf.get(LivyConf.RECOVERY_MODE) == SESSION_RECOVERY_MODE_OFF,
- "Session recovery requires YARN.")
- }
- }
-}
-
-object LivyServer {
-
- def main(args: Array[String]): Unit = {
- val server = new LivyServer()
- try {
- server.start()
- server.join()
- } finally {
- server.stop()
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/SessionServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/SessionServlet.scala b/server/src/main/scala/com/cloudera/livy/server/SessionServlet.scala
deleted file mode 100644
index b08aafa..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/SessionServlet.scala
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-import javax.servlet.http.HttpServletRequest
-
-import org.scalatra._
-import scala.concurrent._
-import scala.concurrent.duration._
-
-import com.cloudera.livy.{LivyConf, Logging}
-import com.cloudera.livy.sessions.{Session, SessionManager}
-import com.cloudera.livy.sessions.Session.RecoveryMetadata
-
-object SessionServlet extends Logging
-
-/**
- * Base servlet for session management. All helper methods in this class assume that the session
- * id parameter in the handler's URI is "id".
- *
- * Type parameters:
- * S: the session type
- */
-abstract class SessionServlet[S <: Session, R <: RecoveryMetadata](
- private[livy] val sessionManager: SessionManager[S, R],
- livyConf: LivyConf)
- extends JsonServlet
- with ApiVersioningSupport
- with MethodOverride
- with UrlGeneratorSupport
- with GZipSupport
-{
- /**
- * Creates a new session based on the current request. The implementation is responsible for
- * parsing the body of the request.
- */
- protected def createSession(req: HttpServletRequest): S
-
- /**
- * Returns a object representing the session data to be sent back to the client.
- */
- protected def clientSessionView(session: S, req: HttpServletRequest): Any = session
-
- override def shutdown(): Unit = {
- sessionManager.shutdown()
- }
-
- before() {
- contentType = "application/json"
- }
-
- get("/") {
- val from = params.get("from").map(_.toInt).getOrElse(0)
- val size = params.get("size").map(_.toInt).getOrElse(100)
-
- val sessions = sessionManager.all()
-
- Map(
- "from" -> from,
- "total" -> sessionManager.size(),
- "sessions" -> sessions.view(from, from + size).map(clientSessionView(_, request))
- )
- }
-
- val getSession = get("/:id") {
- withUnprotectedSession { session =>
- clientSessionView(session, request)
- }
- }
-
- get("/:id/state") {
- withUnprotectedSession { session =>
- Map("id" -> session.id, "state" -> session.state.toString)
- }
- }
-
- get("/:id/log") {
- withSession { session =>
- val from = params.get("from").map(_.toInt)
- val size = params.get("size").map(_.toInt)
- val (from_, total, logLines) = serializeLogs(session, from, size)
-
- Map(
- "id" -> session.id,
- "from" -> from_,
- "total" -> total,
- "log" -> logLines)
- }
- }
-
- delete("/:id") {
- withSession { session =>
- sessionManager.delete(session.id) match {
- case Some(future) =>
- Await.ready(future, Duration.Inf)
- Ok(Map("msg" -> "deleted"))
-
- case None =>
- NotFound(s"Session ${session.id} already stopped.")
- }
- }
- }
-
- post("/") {
- val session = sessionManager.register(createSession(request))
- // Because it may take some time to establish the session, update the last activity
- // time before returning the session info to the client.
- session.recordActivity()
- Created(clientSessionView(session, request),
- headers = Map("Location" ->
- (getRequestPathInfo(request) + url(getSession, "id" -> session.id.toString))))
- }
-
- private def getRequestPathInfo(request: HttpServletRequest): String = {
- if (request.getPathInfo != null && request.getPathInfo != "/") {
- request.getPathInfo
- } else {
- ""
- }
- }
-
- error {
- case e: IllegalArgumentException => BadRequest(e.getMessage)
- }
-
- /**
- * Returns the remote user for the given request. Separate method so that tests can override it.
- */
- protected def remoteUser(req: HttpServletRequest): String = req.getRemoteUser()
-
- /**
- * Checks that the request's user can impersonate the target user.
- *
- * If the user does not have permission to impersonate, then halt execution.
- *
- * @return The user that should be impersonated. That can be the target user if defined, the
- * request's user - which may not be defined - otherwise, or `None` if impersonation is
- * disabled.
- */
- protected def checkImpersonation(
- target: Option[String],
- req: HttpServletRequest): Option[String] = {
- if (livyConf.getBoolean(LivyConf.IMPERSONATION_ENABLED)) {
- if (!target.map(hasAccess(_, req)).getOrElse(true)) {
- halt(Forbidden(s"User '${remoteUser(req)}' not allowed to impersonate '$target'."))
- }
- target.orElse(Option(remoteUser(req)))
- } else {
- None
- }
- }
-
- /**
- * Check that the request's user has access to resources owned by the given target user.
- */
- protected def hasAccess(target: String, req: HttpServletRequest): Boolean = {
- val user = remoteUser(req)
- user == null || user == target || livyConf.superusers().contains(user)
- }
-
- /**
- * Performs an operation on the session, without checking for ownership. Operations executed
- * via this method must not modify the session in any way, or return potentially sensitive
- * information.
- */
- protected def withUnprotectedSession(fn: (S => Any)): Any = doWithSession(fn, true)
-
- /**
- * Performs an operation on the session, verifying whether the caller is the owner of the
- * session.
- */
- protected def withSession(fn: (S => Any)): Any = doWithSession(fn, false)
-
- private def doWithSession(fn: (S => Any), allowAll: Boolean): Any = {
- val sessionId = params("id").toInt
- sessionManager.get(sessionId) match {
- case Some(session) =>
- if (allowAll || hasAccess(session.owner, request)) {
- fn(session)
- } else {
- Forbidden()
- }
- case None =>
- NotFound(s"Session '$sessionId' not found.")
- }
- }
-
- private def serializeLogs(session: S, fromOpt: Option[Int], sizeOpt: Option[Int]) = {
- val lines = session.logLines()
-
- val size = sizeOpt.getOrElse(100)
- var from = fromOpt.getOrElse(-1)
- if (from < 0) {
- from = math.max(0, lines.length - size)
- }
- val until = from + size
-
- (from, lines.length, lines.view(from, until))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/WebServer.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/WebServer.scala b/server/src/main/scala/com/cloudera/livy/server/WebServer.scala
deleted file mode 100644
index 93e32e6..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/WebServer.scala
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-import java.net.InetAddress
-import javax.servlet.ServletContextListener
-
-import org.eclipse.jetty.server._
-import org.eclipse.jetty.server.handler.{HandlerCollection, RequestLogHandler}
-import org.eclipse.jetty.servlet.{DefaultServlet, ServletContextHandler}
-import org.eclipse.jetty.util.ssl.SslContextFactory
-
-import com.cloudera.livy.{LivyConf, Logging}
-
-class WebServer(livyConf: LivyConf, var host: String, var port: Int) extends Logging {
- val server = new Server()
-
- server.setStopTimeout(1000)
- server.setStopAtShutdown(true)
-
- val (connector, protocol) = Option(livyConf.get(LivyConf.SSL_KEYSTORE)) match {
- case None =>
- val http = new HttpConfiguration()
- http.setRequestHeaderSize(livyConf.getInt(LivyConf.REQUEST_HEADER_SIZE))
- http.setResponseHeaderSize(livyConf.getInt(LivyConf.RESPONSE_HEADER_SIZE))
- (new ServerConnector(server, new HttpConnectionFactory(http)), "http")
-
- case Some(keystore) =>
- val https = new HttpConfiguration()
- https.setRequestHeaderSize(livyConf.getInt(LivyConf.REQUEST_HEADER_SIZE))
- https.setResponseHeaderSize(livyConf.getInt(LivyConf.RESPONSE_HEADER_SIZE))
- https.addCustomizer(new SecureRequestCustomizer())
-
- val sslContextFactory = new SslContextFactory()
- sslContextFactory.setKeyStorePath(keystore)
- Option(livyConf.get(LivyConf.SSL_KEYSTORE_PASSWORD))
- .foreach(sslContextFactory.setKeyStorePassword)
- Option(livyConf.get(LivyConf.SSL_KEY_PASSWORD))
- .foreach(sslContextFactory.setKeyManagerPassword)
-
- (new ServerConnector(server,
- new SslConnectionFactory(sslContextFactory, "http/1.1"),
- new HttpConnectionFactory(https)), "https")
- }
-
- connector.setHost(host)
- connector.setPort(port)
-
- server.setConnectors(Array(connector))
-
- val context = new ServletContextHandler()
-
- context.setContextPath("/")
- context.addServlet(classOf[DefaultServlet], "/")
-
- val handlers = new HandlerCollection
- handlers.addHandler(context)
-
- // Configure the access log
- val requestLogHandler = new RequestLogHandler
- val requestLog = new NCSARequestLog(sys.env.getOrElse("LIVY_LOG_DIR",
- sys.env("LIVY_HOME") + "/logs") + "/yyyy_mm_dd.request.log")
- requestLog.setAppend(true)
- requestLog.setExtended(false)
- requestLog.setLogTimeZone("GMT")
- requestLog.setRetainDays(livyConf.getInt(LivyConf.REQUEST_LOG_RETAIN_DAYS))
- requestLogHandler.setRequestLog(requestLog)
- handlers.addHandler(requestLogHandler)
-
- server.setHandler(handlers)
-
- def addEventListener(listener: ServletContextListener): Unit = {
- context.addEventListener(listener)
- }
-
- def start(): Unit = {
- server.start()
-
- val connector = server.getConnectors()(0).asInstanceOf[NetworkConnector]
-
- if (host == "0.0.0.0") {
- host = InetAddress.getLocalHost.getCanonicalHostName
- }
- port = connector.getLocalPort
-
- info("Starting server on %s://%s:%d" format (protocol, host, port))
- }
-
- def join(): Unit = {
- server.join()
- }
-
- def stop(): Unit = {
- context.stop()
- server.stop()
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala b/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
deleted file mode 100644
index c58c99f..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.batch
-
-import java.lang.ProcessBuilder.Redirect
-
-import scala.concurrent.{ExecutionContext, ExecutionContextExecutor}
-import scala.util.Random
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties
-
-import com.cloudera.livy.{LivyConf, Logging}
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.{Session, SessionState}
-import com.cloudera.livy.sessions.Session._
-import com.cloudera.livy.utils.{AppInfo, SparkApp, SparkAppListener, SparkProcessBuilder}
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class BatchRecoveryMetadata(
- id: Int,
- appId: Option[String],
- appTag: String,
- owner: String,
- proxyUser: Option[String],
- version: Int = 1)
- extends RecoveryMetadata
-
-object BatchSession extends Logging {
- val RECOVERY_SESSION_TYPE = "batch"
-
- def create(
- id: Int,
- request: CreateBatchRequest,
- livyConf: LivyConf,
- owner: String,
- proxyUser: Option[String],
- sessionStore: SessionStore,
- mockApp: Option[SparkApp] = None): BatchSession = {
- val appTag = s"livy-batch-$id-${Random.alphanumeric.take(8).mkString}"
-
- def createSparkApp(s: BatchSession): SparkApp = {
- val conf = SparkApp.prepareSparkConf(
- appTag,
- livyConf,
- prepareConf(
- request.conf, request.jars, request.files, request.archives, request.pyFiles, livyConf))
- require(request.file != null, "File is required.")
-
- val builder = new SparkProcessBuilder(livyConf)
- builder.conf(conf)
-
- proxyUser.foreach(builder.proxyUser)
- request.className.foreach(builder.className)
- request.driverMemory.foreach(builder.driverMemory)
- request.driverCores.foreach(builder.driverCores)
- request.executorMemory.foreach(builder.executorMemory)
- request.executorCores.foreach(builder.executorCores)
- request.numExecutors.foreach(builder.numExecutors)
- request.queue.foreach(builder.queue)
- request.name.foreach(builder.name)
-
- // Spark 1.x does not support specifying deploy mode in conf and needs special handling.
- livyConf.sparkDeployMode().foreach(builder.deployMode)
-
- sessionStore.save(BatchSession.RECOVERY_SESSION_TYPE, s.recoveryMetadata)
-
- builder.redirectOutput(Redirect.PIPE)
- builder.redirectErrorStream(true)
-
- val file = resolveURIs(Seq(request.file), livyConf)(0)
- val sparkSubmit = builder.start(Some(file), request.args)
-
- SparkApp.create(appTag, None, Option(sparkSubmit), livyConf, Option(s))
- }
-
- info(s"Creating batch session $id: [owner: $owner, request: $request]")
-
- new BatchSession(
- id,
- appTag,
- SessionState.Starting(),
- livyConf,
- owner,
- proxyUser,
- sessionStore,
- mockApp.map { m => (_: BatchSession) => m }.getOrElse(createSparkApp))
- }
-
- def recover(
- m: BatchRecoveryMetadata,
- livyConf: LivyConf,
- sessionStore: SessionStore,
- mockApp: Option[SparkApp] = None): BatchSession = {
- new BatchSession(
- m.id,
- m.appTag,
- SessionState.Recovering(),
- livyConf,
- m.owner,
- m.proxyUser,
- sessionStore,
- mockApp.map { m => (_: BatchSession) => m }.getOrElse { s =>
- SparkApp.create(m.appTag, m.appId, None, livyConf, Option(s))
- })
- }
-}
-
-class BatchSession(
- id: Int,
- appTag: String,
- initialState: SessionState,
- livyConf: LivyConf,
- owner: String,
- override val proxyUser: Option[String],
- sessionStore: SessionStore,
- sparkApp: BatchSession => SparkApp)
- extends Session(id, owner, livyConf) with SparkAppListener {
- import BatchSession._
-
- protected implicit def executor: ExecutionContextExecutor = ExecutionContext.global
-
- private[this] var _state: SessionState = initialState
- private val app = sparkApp(this)
-
- override def state: SessionState = _state
-
- override def logLines(): IndexedSeq[String] = app.log()
-
- override def stopSession(): Unit = {
- app.kill()
- }
-
- override def appIdKnown(appId: String): Unit = {
- _appId = Option(appId)
- sessionStore.save(RECOVERY_SESSION_TYPE, recoveryMetadata)
- }
-
- override def stateChanged(oldState: SparkApp.State, newState: SparkApp.State): Unit = {
- synchronized {
- debug(s"$this state changed from $oldState to $newState")
- newState match {
- case SparkApp.State.RUNNING =>
- _state = SessionState.Running()
- info(s"Batch session $id created [appid: ${appId.orNull}, state: ${state.toString}, " +
- s"info: ${appInfo.asJavaMap}]")
- case SparkApp.State.FINISHED => _state = SessionState.Success()
- case SparkApp.State.KILLED | SparkApp.State.FAILED =>
- _state = SessionState.Dead()
- case _ =>
- }
- }
- }
-
- override def infoChanged(appInfo: AppInfo): Unit = { this.appInfo = appInfo }
-
- override def recoveryMetadata: RecoveryMetadata =
- BatchRecoveryMetadata(id, appId, appTag, owner, proxyUser)
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/batch/BatchSessionServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/batch/BatchSessionServlet.scala b/server/src/main/scala/com/cloudera/livy/server/batch/BatchSessionServlet.scala
deleted file mode 100644
index ac3a239..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/batch/BatchSessionServlet.scala
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.batch
-
-import javax.servlet.http.HttpServletRequest
-
-import com.cloudera.livy.LivyConf
-import com.cloudera.livy.server.SessionServlet
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.BatchSessionManager
-import com.cloudera.livy.utils.AppInfo
-
-case class BatchSessionView(
- id: Long,
- state: String,
- appId: Option[String],
- appInfo: AppInfo,
- log: Seq[String])
-
-class BatchSessionServlet(
- sessionManager: BatchSessionManager,
- sessionStore: SessionStore,
- livyConf: LivyConf)
- extends SessionServlet(sessionManager, livyConf)
-{
-
- override protected def createSession(req: HttpServletRequest): BatchSession = {
- val createRequest = bodyAs[CreateBatchRequest](req)
- val proxyUser = checkImpersonation(createRequest.proxyUser, req)
- BatchSession.create(
- sessionManager.nextId(), createRequest, livyConf, remoteUser(req), proxyUser, sessionStore)
- }
-
- override protected[batch] def clientSessionView(
- session: BatchSession,
- req: HttpServletRequest): Any = {
- val logs =
- if (hasAccess(session.owner, req)) {
- val lines = session.logLines()
-
- val size = 10
- val from = math.max(0, lines.length - size)
- val until = from + size
-
- lines.view(from, until).toSeq
- } else {
- Nil
- }
- BatchSessionView(session.id, session.state.toString, session.appId, session.appInfo, logs)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala b/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
deleted file mode 100644
index 3a9b49f..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.batch
-
-class CreateBatchRequest {
-
- var file: String = _
- var proxyUser: Option[String] = None
- var args: List[String] = List()
- var className: Option[String] = None
- var jars: List[String] = List()
- var pyFiles: List[String] = List()
- var files: List[String] = List()
- var driverMemory: Option[String] = None
- var driverCores: Option[Int] = None
- var executorMemory: Option[String] = None
- var executorCores: Option[Int] = None
- var numExecutors: Option[Int] = None
- var archives: List[String] = List()
- var queue: Option[String] = None
- var name: Option[String] = None
- var conf: Map[String, String] = Map()
-
- override def toString: String = {
- s"[proxyUser: $proxyUser, " +
- s"file: $file, " +
- (if (args.nonEmpty) s"args: ${args.mkString(",")}, " else "") +
- (if (jars.nonEmpty) s"jars: ${jars.mkString(",")}, " else "") +
- (if (pyFiles.nonEmpty) s"pyFiles: ${pyFiles.mkString(",")}, " else "") +
- (if (files.nonEmpty) s"files: ${files.mkString(",")}, " else "") +
- (if (archives.nonEmpty) s"archives: ${archives.mkString(",")}, " else "") +
- (if (driverMemory.isDefined) s"driverMemory: ${driverMemory.get}, " else "") +
- (if (driverCores.isDefined) s"driverCores: ${driverCores.get}, " else "") +
- (if (executorMemory.isDefined) s"executorMemory: ${executorMemory.get}, " else "") +
- (if (executorCores.isDefined) s"executorCores: ${executorCores.get}, " else "") +
- (if (numExecutors.isDefined) s"numExecutors: ${numExecutors.get}, " else "") +
- (if (queue.isDefined) s"queue: ${queue.get}, " else "") +
- (if (name.isDefined) s"name: ${name.get}, " else "") +
- (if (conf.nonEmpty) s"conf: ${conf.mkString(",")}]" else "]")
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala b/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala
deleted file mode 100644
index 8f861cc..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.interactive
-
-import com.cloudera.livy.sessions.{Kind, Spark}
-
-class CreateInteractiveRequest {
- var kind: Kind = Spark()
- var proxyUser: Option[String] = None
- var jars: List[String] = List()
- var pyFiles: List[String] = List()
- var files: List[String] = List()
- var driverMemory: Option[String] = None
- var driverCores: Option[Int] = None
- var executorMemory: Option[String] = None
- var executorCores: Option[Int] = None
- var numExecutors: Option[Int] = None
- var archives: List[String] = List()
- var queue: Option[String] = None
- var name: Option[String] = None
- var conf: Map[String, String] = Map()
- var heartbeatTimeoutInSecond: Int = 0
-
- override def toString: String = {
- s"[kind: $kind, " +
- s"proxyUser: $proxyUser, " +
- (if (jars.nonEmpty) s"jars: ${jars.mkString(",")}, " else "") +
- (if (pyFiles.nonEmpty) s"pyFiles: ${pyFiles.mkString(",")}, " else "") +
- (if (files.nonEmpty) s"files: ${files.mkString(",")}, " else "") +
- (if (archives.nonEmpty) s"archives: ${archives.mkString(",")}, " else "") +
- (if (driverMemory.isDefined) s"driverMemory: ${driverMemory.get}, " else "") +
- (if (driverCores.isDefined) s"driverCores: ${driverCores.get}, " else "") +
- (if (executorMemory.isDefined) s"executorMemory: ${executorMemory.get}, " else "") +
- (if (executorCores.isDefined) s"executorCores: ${executorCores.get}, " else "") +
- (if (numExecutors.isDefined) s"numExecutors: ${numExecutors.get}, " else "") +
- (if (queue.isDefined) s"queue: ${queue.get}, " else "") +
- (if (name.isDefined) s"name: ${name.get}, " else "") +
- (if (conf.nonEmpty) s"conf: ${conf.mkString(",")}, " else "") +
- s"heartbeatTimeoutInSecond: $heartbeatTimeoutInSecond]"
- }
-}
[28/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/main/scala/org/apache/livy/test/framework/RealCluster.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/RealCluster.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/RealCluster.scala
new file mode 100644
index 0000000..0acc580
--- /dev/null
+++ b/integration-test/src/main/scala/org/apache/livy/test/framework/RealCluster.scala
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.framework
+
+import java.io.{File, IOException}
+import java.nio.file.Files
+import java.security.PrivilegedExceptionAction
+import javax.servlet.http.HttpServletResponse._
+
+import scala.collection.JavaConverters._
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.sys.process._
+import scala.util.Random
+
+import com.decodified.scalassh._
+import com.ning.http.client.AsyncHttpClient
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.hadoop.security.UserGroupInformation
+import org.scalatest.concurrent.Eventually._
+
+import org.apache.livy.{LivyConf, Logging}
+
+private class RealClusterConfig(config: Map[String, String]) {
+ val ip = config("ip")
+
+ val sshLogin = config("ssh.login")
+ val sshPubKey = config("ssh.pubkey")
+ val livyPort = config.getOrElse(LivyConf.SERVER_PORT.key, "8998").toInt
+ val livyClasspath = config.getOrElse("livy.classpath", "")
+
+ val deployLivy = config.getOrElse("deploy-livy", "true").toBoolean
+ val noDeployLivyHome = config.get("livy-home")
+
+ val sparkHome = config("env.spark_home")
+ val sparkConf = config.getOrElse("env.spark_conf", "/etc/spark/conf")
+ val hadoopConf = config.getOrElse("env.hadoop_conf", "/etc/hadoop/conf")
+
+ val javaHome = config.getOrElse("env.java_home", "/usr/java/default")
+}
+
+class RealCluster(_config: Map[String, String])
+ extends Cluster with ClusterUtils with Logging {
+
+ private val config = new RealClusterConfig(_config)
+
+ private var livyIsRunning = false
+ private var livyHomePath: String = _
+ private var livyEpoch = 0
+
+ private var _configDir: File = _
+
+ private var hdfsScratch: Path = _
+
+ private var sparkConfDir: String = _
+ private var tempDirPath: String = _
+ private var _hasSparkR: Boolean = _
+
+ override def isRealSpark(): Boolean = true
+
+ override def hasSparkR(): Boolean = _hasSparkR
+
+ override def configDir(): File = _configDir
+
+ override def hdfsScratchDir(): Path = hdfsScratch
+
+ override def doAsClusterUser[T](task: => T): T = {
+ val user = UserGroupInformation.createRemoteUser(config.sshLogin)
+ user.doAs(new PrivilegedExceptionAction[T] {
+ override def run(): T = task
+ })
+ }
+
+ private def sshClient[T](body: SshClient => SSH.Result[T]): T = {
+ val sshLogin = PublicKeyLogin(
+ config.sshLogin, None, config.sshPubKey :: Nil)
+ val hostConfig = HostConfig(login = sshLogin, hostKeyVerifier = HostKeyVerifiers.DontVerify)
+ SSH(config.ip, hostConfig)(body) match {
+ case Left(err) => throw new IOException(err)
+ case Right(result) => result
+ }
+ }
+
+ private def exec(cmd: String): CommandResult = {
+ info(s"Running command: $cmd")
+ val result = sshClient(_.exec(cmd))
+ result.exitCode match {
+ case Some(ec) if ec > 0 =>
+ throw new IOException(s"Command '$cmd' failed: $ec\n" +
+ s"stdout: ${result.stdOutAsString()}\n" +
+ s"stderr: ${result.stdErrAsString()}\n")
+ case _ =>
+ }
+ result
+ }
+
+ private def upload(local: String, remote: String): Unit = {
+ info(s"Uploading local path $local")
+ sshClient(_.upload(local, remote))
+ }
+
+ private def download(remote: String, local: String): Unit = {
+ info(s"Downloading remote path $remote")
+ sshClient(_.download(remote, local))
+ }
+
+ override def deploy(): Unit = {
+ // Make sure Livy is not running.
+ stopLivy()
+
+ // Check whether SparkR is supported in YARN (need the sparkr.zip archive).\
+ _hasSparkR = try {
+ exec(s"test -f ${config.sparkHome}/R/lib/sparkr.zip")
+ true
+ } catch {
+ case e: IOException => false
+ }
+
+ // Copy the remove Hadoop configuration to a local temp dir so that tests can use it to
+ // talk to HDFS and YARN.
+ val localTemp = new File(sys.props("java.io.tmpdir") + File.separator + "hadoop-conf")
+ download(config.hadoopConf, localTemp.getAbsolutePath())
+ _configDir = localTemp
+
+ // Create a temp directory where test files will be written.
+ tempDirPath = s"/tmp/livy-it-${Random.alphanumeric.take(16).mkString}"
+ exec(s"mkdir -p $tempDirPath")
+
+ // Also create an HDFS scratch directory for tests.
+ doAsClusterUser {
+ hdfsScratch = fs.makeQualified(
+ new Path(s"/tmp/livy-it-${Random.alphanumeric.take(16).mkString}"))
+ fs.mkdirs(hdfsScratch)
+ fs.setPermission(hdfsScratch, new FsPermission("777"))
+ }
+
+ // Create a copy of the Spark configuration, and make sure the master is "yarn-cluster".
+ sparkConfDir = s"$tempDirPath/spark-conf"
+ val sparkProps = s"$sparkConfDir/spark-defaults.conf"
+ Seq(
+ s"cp -Lr ${config.sparkConf} $sparkConfDir",
+ s"touch $sparkProps",
+ s"sed -i.old '/spark.master.*/d' $sparkProps",
+ s"sed -i.old '/spark.submit.deployMode.*/d' $sparkProps",
+ s"echo 'spark.master=yarn-cluster' >> $sparkProps"
+ ).foreach(exec)
+
+ if (config.deployLivy) {
+ try {
+ info(s"Deploying Livy to ${config.ip}...")
+ val version = sys.props("project.version")
+ val assemblyZip = new File(s"../assembly/target/livy-server-$version.zip")
+ assert(assemblyZip.isFile,
+ s"Can't find livy assembly zip at ${assemblyZip.getCanonicalPath}")
+ val assemblyName = assemblyZip.getName()
+
+ // SSH to the node to unzip and install Livy.
+ upload(assemblyZip.getCanonicalPath, s"$tempDirPath/$assemblyName")
+ exec(s"unzip -o $tempDirPath/$assemblyName -d $tempDirPath")
+ livyHomePath = s"$tempDirPath/livy-server-$version"
+ info(s"Deployed Livy to ${config.ip} at $livyHomePath.")
+ } catch {
+ case e: Exception =>
+ error(s"Failed to deploy Livy to ${config.ip}.", e)
+ cleanUp()
+ throw e
+ }
+ } else {
+ livyHomePath = config.noDeployLivyHome.get
+ info("Skipping deployment.")
+ }
+
+ runLivy()
+ }
+
+ override def cleanUp(): Unit = {
+ stopLivy()
+ if (tempDirPath != null) {
+ exec(s"rm -rf $tempDirPath")
+ }
+ if (hdfsScratch != null) {
+ doAsClusterUser {
+ // Cannot use the shared `fs` since this runs in a shutdown hook, and that instance
+ // may have been closed already.
+ val fs = FileSystem.newInstance(hadoopConf)
+ try {
+ fs.delete(hdfsScratch, true)
+ } finally {
+ fs.close()
+ }
+ }
+ }
+ }
+
+ override def runLivy(): Unit = synchronized {
+ assert(!livyIsRunning, "Livy is running already.")
+
+ val livyConf = Map(
+ "livy.server.port" -> config.livyPort.toString,
+ // "livy.server.recovery.mode=local",
+ "livy.environment" -> "development",
+ LivyConf.LIVY_SPARK_MASTER.key -> "yarn",
+ LivyConf.LIVY_SPARK_DEPLOY_MODE.key -> "cluster")
+ val livyConfFile = File.createTempFile("livy.", ".properties")
+ saveProperties(livyConf, livyConfFile)
+ upload(livyConfFile.getAbsolutePath(), s"$livyHomePath/conf/livy.conf")
+
+ val env = Map(
+ "JAVA_HOME" -> config.javaHome,
+ "HADOOP_CONF_DIR" -> config.hadoopConf,
+ "SPARK_CONF_DIR" -> sparkConfDir,
+ "SPARK_HOME" -> config.sparkHome,
+ "CLASSPATH" -> config.livyClasspath,
+ "LIVY_PID_DIR" -> s"$tempDirPath/pid",
+ "LIVY_LOG_DIR" -> s"$tempDirPath/logs",
+ "LIVY_MAX_LOG_FILES" -> "16",
+ "LIVY_IDENT_STRING" -> "it"
+ )
+ val livyEnvFile = File.createTempFile("livy-env.", ".sh")
+ saveProperties(env, livyEnvFile)
+ upload(livyEnvFile.getAbsolutePath(), s"$livyHomePath/conf/livy-env.sh")
+
+ info(s"Starting Livy @ port ${config.livyPort}...")
+ exec(s"env -i $livyHomePath/bin/livy-server start")
+ livyIsRunning = true
+
+ val httpClient = new AsyncHttpClient()
+ eventually(timeout(1 minute), interval(1 second)) {
+ assert(httpClient.prepareGet(livyEndpoint).execute().get().getStatusCode == SC_OK)
+ }
+ info(s"Started Livy.")
+ }
+
+ override def stopLivy(): Unit = synchronized {
+ info("Stopping Livy Server")
+ try {
+ exec(s"$livyHomePath/bin/livy-server stop")
+ } catch {
+ case e: Exception =>
+ if (livyIsRunning) {
+ throw e
+ }
+ }
+
+ if (livyIsRunning) {
+ // Wait a tiny bit so that the process finishes closing its output files.
+ Thread.sleep(2)
+
+ livyEpoch += 1
+ val logName = "livy-it-server.out"
+ val localName = s"livy-it-server-$livyEpoch.out"
+ val logPath = s"$tempDirPath/logs/$logName"
+ val localLog = sys.props("java.io.tmpdir") + File.separator + localName
+ download(logPath, localLog)
+ info(s"Log for epoch $livyEpoch available at $localLog")
+ }
+ }
+
+ override def livyEndpoint: String = s"http://${config.ip}:${config.livyPort}"
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/test/scala/com/cloudera/livy/test/BatchIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/scala/com/cloudera/livy/test/BatchIT.scala b/integration-test/src/test/scala/com/cloudera/livy/test/BatchIT.scala
deleted file mode 100644
index 179239a..0000000
--- a/integration-test/src/test/scala/com/cloudera/livy/test/BatchIT.scala
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test
-
-import java.io.File
-import java.util.UUID
-
-import scala.language.postfixOps
-
-import org.apache.commons.io.IOUtils
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
-import org.scalatest.BeforeAndAfterAll
-import org.scalatest.OptionValues._
-
-import com.cloudera.livy.sessions.SessionState
-import com.cloudera.livy.test.apps._
-import com.cloudera.livy.test.framework.{BaseIntegrationTestSuite, LivyRestClient}
-
-class BatchIT extends BaseIntegrationTestSuite with BeforeAndAfterAll {
- private var testLibPath: String = _
-
- protected override def beforeAll() = {
- super.beforeAll()
- testLibPath = uploadToHdfs(new File(testLib))
- }
-
- test("submit spark app") {
- val output = newOutputPath()
- withTestLib(classOf[SimpleSparkApp], List(output)) { s =>
- s.verifySessionSuccess()
-
- // Make sure the test lib has created the test output.
- cluster.fs.isDirectory(new Path(output)) shouldBe true
-
- // Make sure appInfo is reported correctly.
- val state = s.snapshot()
- state.appInfo.sparkUiUrl.value should startWith ("http")
- }
- }
-
- test("submit an app that fails") {
- val output = newOutputPath()
- withTestLib(classOf[FailingApp], List(output)) { s =>
- // At this point the application has exited. State should be 'dead' instead of 'error'.
- s.verifySessionDead()
-
- // The file is written to make sure the app actually ran, instead of just failing for
- // some other reason.
- cluster.fs.isFile(new Path(output)) shouldBe true
- }
- }
-
- pytest("submit a pyspark application") {
- val scriptPath = uploadResource("batch.py")
- val output = newOutputPath()
- withScript(scriptPath, List(output)) { s =>
- s.verifySessionSuccess()
- cluster.fs.isDirectory(new Path(output)) shouldBe true
- }
- }
-
- // This is disabled since R scripts don't seem to work in yarn-cluster mode. There's a
- // TODO comment in Spark's ApplicationMaster.scala.
- ignore("submit a SparkR application") {
- val hdfsPath = uploadResource("rtest.R")
- withScript(hdfsPath, List.empty) { s =>
- s.verifySessionSuccess()
- }
- }
-
- test("deleting a session should kill YARN app") {
- val output = newOutputPath()
- withTestLib(classOf[SimpleSparkApp], List(output, "false")) { s =>
- s.verifySessionState(SessionState.Running())
- s.snapshot().appInfo.driverLogUrl.value should include ("containerlogs")
-
- val appId = s.appId()
-
- // Delete the session then verify the YARN app state is KILLED.
- s.stop()
- cluster.yarnClient.getApplicationReport(appId).getFinalApplicationStatus shouldBe
- FinalApplicationStatus.KILLED
- }
- }
-
- test("killing YARN app should change batch state to dead") {
- val output = newOutputPath()
- withTestLib(classOf[SimpleSparkApp], List(output, "false")) { s =>
- s.verifySessionState(SessionState.Running())
- val appId = s.appId()
-
- // Kill the YARN app and check batch state should be KILLED.
- cluster.yarnClient.killApplication(appId)
- s.verifySessionDead()
-
- cluster.yarnClient.getApplicationReport(appId).getFinalApplicationStatus shouldBe
- FinalApplicationStatus.KILLED
-
- s.snapshot().log should contain ("Application killed by user.")
- }
- }
-
- test("recover batch sessions") {
- val output1 = newOutputPath()
- val output2 = newOutputPath()
- withTestLib(classOf[SimpleSparkApp], List(output1)) { s1 =>
- s1.stop()
- withTestLib(classOf[SimpleSparkApp], List(output2, "false")) { s2 =>
- s2.verifySessionRunning()
-
- restartLivy()
-
- // Verify previous active session still appears after restart.
- s2.verifySessionRunning()
- // Verify deleted session doesn't show up.
- s1.verifySessionDoesNotExist()
-
- s2.stop()
-
- // Verify new session doesn't reuse old session id.
- withTestLib(classOf[SimpleSparkApp], List(output2)) { s3 =>
- s3.id should be > s2.id
- }
- }
- }
- }
-
- private def newOutputPath(): String = {
- cluster.hdfsScratchDir().toString() + "/" + UUID.randomUUID().toString()
- }
-
- private def uploadResource(name: String): String = {
- val hdfsPath = new Path(cluster.hdfsScratchDir(), UUID.randomUUID().toString + "-" + name)
- val in = getClass.getResourceAsStream("/" + name)
- val out = cluster.fs.create(hdfsPath)
- try {
- IOUtils.copy(in, out)
- } finally {
- in.close()
- out.close()
- }
- hdfsPath.toUri().getPath()
- }
-
- private def withScript[R]
- (scriptPath: String, args: List[String], sparkConf: Map[String, String] = Map.empty)
- (f: (LivyRestClient#BatchSession) => R): R = {
- val s = livyClient.startBatch(scriptPath, None, args, sparkConf)
- withSession(s)(f)
- }
-
- private def withTestLib[R]
- (testClass: Class[_], args: List[String], sparkConf: Map[String, String] = Map.empty)
- (f: (LivyRestClient#BatchSession) => R): R = {
- val s = livyClient.startBatch(testLibPath, Some(testClass.getName()), args, sparkConf)
- withSession(s)(f)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/test/scala/com/cloudera/livy/test/InteractiveIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/scala/com/cloudera/livy/test/InteractiveIT.scala b/integration-test/src/test/scala/com/cloudera/livy/test/InteractiveIT.scala
deleted file mode 100644
index 88a995c..0000000
--- a/integration-test/src/test/scala/com/cloudera/livy/test/InteractiveIT.scala
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test
-
-import java.util.concurrent.atomic.AtomicInteger
-import java.util.regex.Pattern
-
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.scalatest.concurrent.Eventually._
-import org.scalatest.OptionValues._
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import com.cloudera.livy.rsc.RSCConf
-import com.cloudera.livy.sessions._
-import com.cloudera.livy.test.framework.{BaseIntegrationTestSuite, LivyRestClient}
-
-class InteractiveIT extends BaseIntegrationTestSuite {
- test("basic interactive session") {
- withNewSession(Spark()) { s =>
- s.run("val sparkVersion = sc.version").result().left.foreach(info(_))
- s.run("1+1").verifyResult("res0: Int = 2")
- s.run("""sc.getConf.get("spark.executor.instances")""").verifyResult("res1: String = 1")
- s.run("val sql = new org.apache.spark.sql.SQLContext(sc)").verifyResult(
- ".*" + Pattern.quote(
- "sql: org.apache.spark.sql.SQLContext = org.apache.spark.sql.SQLContext") + ".*")
- s.run("abcde").verifyError(evalue = ".*?:[0-9]+: error: not found: value abcde.*")
- s.run("throw new IllegalStateException()")
- .verifyError(evalue = ".*java\\.lang\\.IllegalStateException.*")
-
- // Verify Livy internal configurations are not exposed.
- // TODO separate all these checks to different sub tests after merging new IT code.
- s.run("""sc.getConf.getAll.exists(_._1.startsWith("spark.__livy__."))""")
- .verifyResult(".*false")
- s.run("""sys.props.exists(_._1.startsWith("spark.__livy__."))""").verifyResult(".*false")
-
- // Make sure appInfo is reported correctly.
- val state = s.snapshot()
- state.appInfo.driverLogUrl.value should include ("containerlogs")
- state.appInfo.sparkUiUrl.value should startWith ("http")
-
- // Stop session and verify the YARN app state is finished.
- // This is important because if YARN app state is killed, Spark history is not archived.
- val appId = s.appId()
- s.stop()
- val appReport = cluster.yarnClient.getApplicationReport(appId)
- appReport.getYarnApplicationState() shouldEqual YarnApplicationState.FINISHED
- }
- }
-
- pytest("pyspark interactive session") {
- withNewSession(PySpark()) { s =>
- s.run("1+1").verifyResult("2")
- s.run("sqlContext").verifyResult(startsWith("<pyspark.sql.context.HiveContext"))
- s.run("sc.parallelize(range(100)).map(lambda x: x * 2).reduce(lambda x, y: x + y)")
- .verifyResult("9900")
- s.run("from pyspark.sql.types import Row").verifyResult("")
- s.run("x = [Row(age=1, name=u'a'), Row(age=2, name=u'b'), Row(age=3, name=u'c')]")
- .verifyResult("")
- s.run("%table x").verifyResult(".*headers.*type.*name.*data.*")
- s.run("abcde").verifyError(ename = "NameError", evalue = "name 'abcde' is not defined")
- s.run("raise KeyError, 'foo'").verifyError(ename = "KeyError", evalue = "'foo'")
- }
- }
-
- rtest("R interactive session") {
- withNewSession(SparkR()) { s =>
- // R's output sometimes includes the count of statements, which makes it annoying to test
- // things. This helps a bit.
- val curr = new AtomicInteger()
- def count: Int = curr.incrementAndGet()
-
- s.run("1+1").verifyResult(startsWith(s"[$count] 2"))
- s.run("sqlContext <- sparkRSQL.init(sc)").verifyResult(null)
- s.run("hiveContext <- sparkRHive.init(sc)").verifyResult(null)
- s.run("""localDF <- data.frame(name=c("John", "Smith", "Sarah"), age=c(19, 23, 18))""")
- .verifyResult(null)
- s.run("df <- createDataFrame(sqlContext, localDF)").verifyResult(null)
- s.run("printSchema(df)").verifyResult(literal(
- """|root
- | |-- name: string (nullable = true)
- | |-- age: double (nullable = true)""".stripMargin))
- }
- }
-
- test("application kills session") {
- withNewSession(Spark()) { s =>
- s.runFatalStatement("System.exit(0)")
- }
- }
-
- test("should kill RSCDriver if it doesn't respond to end session") {
- val testConfName = s"${RSCConf.LIVY_SPARK_PREFIX}${RSCConf.Entry.TEST_STUCK_END_SESSION.key()}"
- withNewSession(Spark(), Map(testConfName -> "true")) { s =>
- val appId = s.appId()
- s.stop()
- val appReport = cluster.yarnClient.getApplicationReport(appId)
- appReport.getYarnApplicationState() shouldBe YarnApplicationState.KILLED
- }
- }
-
- test("should kill RSCDriver if it didn't register itself in time") {
- val testConfName =
- s"${RSCConf.LIVY_SPARK_PREFIX}${RSCConf.Entry.TEST_STUCK_START_DRIVER.key()}"
- withNewSession(Spark(), Map(testConfName -> "true"), false) { s =>
- eventually(timeout(2 minutes), interval(5 seconds)) {
- val appId = s.appId()
- appId should not be null
- val appReport = cluster.yarnClient.getApplicationReport(appId)
- appReport.getYarnApplicationState() shouldBe YarnApplicationState.KILLED
- }
- }
- }
-
- test("user jars are properly imported in Scala interactive sessions") {
- // Include a popular Java library to test importing user jars.
- val sparkConf = Map("spark.jars.packages" -> "org.codehaus.plexus:plexus-utils:3.0.24")
- withNewSession(Spark(), sparkConf) { s =>
- // Check is the library loaded in JVM in the proper class loader.
- s.run("Thread.currentThread.getContextClassLoader.loadClass" +
- """("org.codehaus.plexus.util.FileUtils")""")
- .verifyResult(".*Class\\[_\\] = class org.codehaus.plexus.util.FileUtils")
-
- // Check does Scala interpreter see the library.
- s.run("import org.codehaus.plexus.util._").verifyResult("import org.codehaus.plexus.util._")
-
- // Check does SparkContext see classes defined by Scala interpreter.
- s.run("case class Item(i: Int)").verifyResult("defined class Item")
- s.run("val rdd = sc.parallelize(Array.fill(10){new Item(scala.util.Random.nextInt(1000))})")
- .verifyResult("rdd.*")
- s.run("rdd.count()").verifyResult(".*= 10")
- }
- }
-
- test("heartbeat should kill expired session") {
- // Set it to 2s because verifySessionIdle() is calling GET every second.
- val heartbeatTimeout = Duration.create("2s")
- withNewSession(Spark(), Map.empty, true, heartbeatTimeout.toSeconds.toInt) { s =>
- // If the test reaches here, that means verifySessionIdle() is successfully keeping the
- // session alive. Now verify heartbeat is killing expired session.
- Thread.sleep(heartbeatTimeout.toMillis * 2)
- s.verifySessionDoesNotExist()
- }
- }
-
- test("recover interactive session") {
- withNewSession(Spark()) { s =>
- val stmt1 = s.run("1")
- stmt1.verifyResult("res0: Int = 1")
-
- restartLivy()
-
- // Verify session still exists.
- s.verifySessionIdle()
- s.run("2").verifyResult("res1: Int = 2")
- // Verify statement result is preserved.
- stmt1.verifyResult("res0: Int = 1")
-
- s.stop()
-
- restartLivy()
-
- // Verify deleted session doesn't show up after recovery.
- s.verifySessionDoesNotExist()
-
- // Verify new session doesn't reuse old session id.
- withNewSession(Spark(), Map.empty, false) { s1 =>
- s1.id should be > s.id
- }
- }
- }
-
- private def withNewSession[R] (
- kind: Kind,
- sparkConf: Map[String, String] = Map.empty,
- waitForIdle: Boolean = true,
- heartbeatTimeoutInSecond: Int = 0)
- (f: (LivyRestClient#InteractiveSession) => R): R = {
- withSession(livyClient.startSession(kind, sparkConf, heartbeatTimeoutInSecond)) { s =>
- if (waitForIdle) {
- s.verifySessionIdle()
- }
- f(s)
- }
- }
-
- private def startsWith(result: String): String = Pattern.quote(result) + ".*"
-
- private def literal(result: String): String = Pattern.quote(result)
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/test/scala/com/cloudera/livy/test/JobApiIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/scala/com/cloudera/livy/test/JobApiIT.scala b/integration-test/src/test/scala/com/cloudera/livy/test/JobApiIT.scala
deleted file mode 100644
index 87dcfe1..0000000
--- a/integration-test/src/test/scala/com/cloudera/livy/test/JobApiIT.scala
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test
-
-import java.io.{File, InputStream}
-import java.net.URI
-import java.nio.charset.StandardCharsets.UTF_8
-import java.nio.file.{Files, StandardCopyOption}
-import java.util.concurrent.{Future => JFuture, TimeUnit}
-import javax.servlet.http.HttpServletResponse
-
-import scala.collection.JavaConverters._
-import scala.util.{Properties, Try}
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.module.scala.DefaultScalaModule
-import org.scalatest.BeforeAndAfterAll
-
-import com.cloudera.livy._
-import com.cloudera.livy.client.common.HttpMessages._
-import com.cloudera.livy.sessions.SessionKindModule
-import com.cloudera.livy.test.framework.BaseIntegrationTestSuite
-import com.cloudera.livy.test.jobs._
-import com.cloudera.livy.utils.LivySparkUtils
-
-// Proper type representing the return value of "GET /sessions". At some point we should make
-// SessionServlet use something like this.
-class SessionList {
- val from: Int = -1
- val total: Int = -1
- val sessions: List[SessionInfo] = Nil
-}
-
-class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logging {
-
- private var client: LivyClient = _
- private var sessionId: Int = _
- private var client2: LivyClient = _
- private val mapper = new ObjectMapper()
- .registerModule(DefaultScalaModule)
- .registerModule(new SessionKindModule())
-
- override def afterAll(): Unit = {
- super.afterAll()
- Seq(client, client2).foreach { c =>
- if (c != null) {
- c.stop(true)
- }
- }
-
- livyClient.connectSession(sessionId).stop()
- }
-
- test("create a new session and upload test jar") {
- val tempClient = createClient(livyEndpoint)
-
- try {
- // Figure out the session ID by poking at the REST endpoint. We should probably expose this
- // in the Java API.
- val list = sessionList()
- assert(list.total === 1)
- val tempSessionId = list.sessions(0).id
-
- livyClient.connectSession(tempSessionId).verifySessionIdle()
- waitFor(tempClient.uploadJar(new File(testLib)))
-
- client = tempClient
- sessionId = tempSessionId
- } finally {
- if (client == null) {
- try {
- tempClient.stop(true)
- } catch {
- case e: Exception => warn("Error stopping client.", e)
- }
- }
- }
- }
-
- test("upload file") {
- assume(client != null, "Client not active.")
-
- val file = Files.createTempFile("filetest", ".txt")
- Files.write(file, "hello".getBytes(UTF_8))
-
- waitFor(client.uploadFile(file.toFile()))
-
- val result = waitFor(client.submit(new FileReader(file.toFile().getName(), false)))
- assert(result === "hello")
- }
-
- test("add file from HDFS") {
- assume(client != null, "Client not active.")
- val file = Files.createTempFile("filetest2", ".txt")
- Files.write(file, "hello".getBytes(UTF_8))
-
- val uri = new URI(uploadToHdfs(file.toFile()))
- waitFor(client.addFile(uri))
-
- val task = new FileReader(new File(uri.getPath()).getName(), false)
- val result = waitFor(client.submit(task))
- assert(result === "hello")
- }
-
- test("run simple jobs") {
- assume(client != null, "Client not active.")
-
- val result = waitFor(client.submit(new Echo("hello")))
- assert(result === "hello")
-
- val result2 = waitFor(client.run(new Echo("hello")))
- assert(result2 === "hello")
- }
-
- test("run spark job") {
- assume(client != null, "Client not active.")
- val result = waitFor(client.submit(new SmallCount(100)))
- assert(result === 100)
- }
-
- test("run spark sql job") {
- assume(client != null, "Client not active.")
- val result = waitFor(client.submit(new SQLGetTweets(false)))
- assert(result.size() > 0)
- }
-
- test("stop a client without destroying the session") {
- assume(client != null, "Client not active.")
- client.stop(false)
- client = null
- }
-
- test("connect to an existing session") {
- livyClient.connectSession(sessionId).verifySessionIdle()
- val sessionUri = s"$livyEndpoint/sessions/$sessionId"
- client2 = createClient(sessionUri)
- }
-
- test("submit job using new client") {
- assume(client2 != null, "Client not active.")
- val result = waitFor(client2.submit(new Echo("hello")))
- assert(result === "hello")
- }
-
- scalaTest("run scala jobs") {
- assume(client2 != null, "Client not active.")
-
- val jobs = Seq(
- new ScalaEcho("abcde"),
- new ScalaEcho(Seq(1, 2, 3, 4)),
- new ScalaEcho(Map(1 -> 2, 3 -> 4)),
- new ScalaEcho(ValueHolder("abcde")),
- new ScalaEcho(ValueHolder(Seq(1, 2, 3, 4))),
- new ScalaEcho(Some("abcde"))
- )
-
- jobs.foreach { job =>
- val result = waitFor(client2.submit(job))
- assert(result === job.value)
- }
- }
-
- protected def scalaTest(desc: String)(testFn: => Unit): Unit = {
- test(desc) {
- assume(sys.env("LIVY_SPARK_SCALA_VERSION") ==
- LivySparkUtils.formatScalaVersion(Properties.versionNumberString),
- s"Scala test can only be run with ${Properties.versionString}")
- testFn
- }
- }
-
- test("ensure failing jobs do not affect session state") {
- assume(client2 != null, "Client not active.")
-
- try {
- waitFor(client2.submit(new Failure()))
- fail("Job should have failued.")
- } catch {
- case e: Exception =>
- assert(e.getMessage().contains(classOf[Failure.JobFailureException].getName()))
- }
-
- val result = waitFor(client2.submit(new Echo("foo")))
- assert(result === "foo")
- }
-
- test("return null should not throw NPE") {
- assume(client2 != null, "Client not active")
-
- val result = waitFor(client2.submit(new VoidJob()))
- assert(result === null)
- }
-
- test("destroy the session") {
- assume(client2 != null, "Client not active.")
- client2.stop(true)
-
- val list = sessionList()
- assert(list.total === 0)
-
- val sessionUri = s"$livyEndpoint/sessions/$sessionId"
- Try(createClient(sessionUri)).foreach { client =>
- client.stop(true)
- fail("Should not have been able to connect to destroyed session.")
- }
-
- sessionId = -1
- }
-
- pytest("validate Python-API requests") {
- val addFileContent = "hello from addfile"
- val addFilePath = createTempFilesForTest("add_file", ".txt", addFileContent, true)
- val addPyFileContent = "def test_add_pyfile(): return \"hello from addpyfile\""
- val addPyFilePath = createTempFilesForTest("add_pyfile", ".py", addPyFileContent, true)
- val uploadFileContent = "hello from uploadfile"
- val uploadFilePath = createTempFilesForTest("upload_pyfile", ".py", uploadFileContent, false)
- val uploadPyFileContent = "def test_upload_pyfile(): return \"hello from uploadpyfile\""
- val uploadPyFilePath = createTempFilesForTest("upload_pyfile", ".py",
- uploadPyFileContent, false)
-
- val builder = new ProcessBuilder(Seq("python", createPyTestsForPythonAPI().toString).asJava)
-
- val env = builder.environment()
- env.put("LIVY_END_POINT", livyEndpoint)
- env.put("ADD_FILE_URL", addFilePath)
- env.put("ADD_PYFILE_URL", addPyFilePath)
- env.put("UPLOAD_FILE_URL", uploadFilePath)
- env.put("UPLOAD_PYFILE_URL", uploadPyFilePath)
-
- builder.redirectOutput(new File(sys.props("java.io.tmpdir") + "/pytest_results.txt"))
- builder.redirectErrorStream(true)
-
- val process = builder.start()
-
- process.waitFor()
-
- assert(process.exitValue() === 0)
- }
-
- private def createPyTestsForPythonAPI(): File = {
- var source: InputStream = null
- try {
- source = getClass.getClassLoader.getResourceAsStream("test_python_api.py")
- val file = Files.createTempFile("", "").toFile
- Files.copy(source, file.toPath, StandardCopyOption.REPLACE_EXISTING)
- file
- } finally {
- source.close()
- }
- }
-
- private def createTempFilesForTest(
- fileName: String,
- fileExtension: String,
- fileContent: String,
- uploadFileToHdfs: Boolean): String = {
- val path = Files.createTempFile(fileName, fileExtension)
- Files.write(path, fileContent.getBytes(UTF_8))
- if (uploadFileToHdfs) {
- uploadToHdfs(path.toFile())
- } else {
- path.toString
- }
- }
-
- private def waitFor[T](future: JFuture[T]): T = {
- future.get(30, TimeUnit.SECONDS)
- }
-
- private def sessionList(): SessionList = {
- val response = httpClient.prepareGet(s"$livyEndpoint/sessions/").execute().get()
- assert(response.getStatusCode === HttpServletResponse.SC_OK)
- mapper.readValue(response.getResponseBodyAsStream, classOf[SessionList])
- }
-
- private def createClient(uri: String): LivyClient = {
- new LivyClientBuilder().setURI(new URI(uri)).build()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/test/scala/org/apache/livy/test/BatchIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/scala/org/apache/livy/test/BatchIT.scala b/integration-test/src/test/scala/org/apache/livy/test/BatchIT.scala
new file mode 100644
index 0000000..0b3a061
--- /dev/null
+++ b/integration-test/src/test/scala/org/apache/livy/test/BatchIT.scala
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test
+
+import java.io.File
+import java.util.UUID
+
+import scala.language.postfixOps
+
+import org.apache.commons.io.IOUtils
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.OptionValues._
+
+import org.apache.livy.sessions.SessionState
+import org.apache.livy.test.apps._
+import org.apache.livy.test.framework.{BaseIntegrationTestSuite, LivyRestClient}
+
+class BatchIT extends BaseIntegrationTestSuite with BeforeAndAfterAll {
+ private var testLibPath: String = _
+
+ protected override def beforeAll() = {
+ super.beforeAll()
+ testLibPath = uploadToHdfs(new File(testLib))
+ }
+
+ test("submit spark app") {
+ val output = newOutputPath()
+ withTestLib(classOf[SimpleSparkApp], List(output)) { s =>
+ s.verifySessionSuccess()
+
+ // Make sure the test lib has created the test output.
+ cluster.fs.isDirectory(new Path(output)) shouldBe true
+
+ // Make sure appInfo is reported correctly.
+ val state = s.snapshot()
+ state.appInfo.sparkUiUrl.value should startWith ("http")
+ }
+ }
+
+ test("submit an app that fails") {
+ val output = newOutputPath()
+ withTestLib(classOf[FailingApp], List(output)) { s =>
+ // At this point the application has exited. State should be 'dead' instead of 'error'.
+ s.verifySessionDead()
+
+ // The file is written to make sure the app actually ran, instead of just failing for
+ // some other reason.
+ cluster.fs.isFile(new Path(output)) shouldBe true
+ }
+ }
+
+ pytest("submit a pyspark application") {
+ val scriptPath = uploadResource("batch.py")
+ val output = newOutputPath()
+ withScript(scriptPath, List(output)) { s =>
+ s.verifySessionSuccess()
+ cluster.fs.isDirectory(new Path(output)) shouldBe true
+ }
+ }
+
+ // This is disabled since R scripts don't seem to work in yarn-cluster mode. There's a
+ // TODO comment in Spark's ApplicationMaster.scala.
+ ignore("submit a SparkR application") {
+ val hdfsPath = uploadResource("rtest.R")
+ withScript(hdfsPath, List.empty) { s =>
+ s.verifySessionSuccess()
+ }
+ }
+
+ test("deleting a session should kill YARN app") {
+ val output = newOutputPath()
+ withTestLib(classOf[SimpleSparkApp], List(output, "false")) { s =>
+ s.verifySessionState(SessionState.Running())
+ s.snapshot().appInfo.driverLogUrl.value should include ("containerlogs")
+
+ val appId = s.appId()
+
+ // Delete the session then verify the YARN app state is KILLED.
+ s.stop()
+ cluster.yarnClient.getApplicationReport(appId).getFinalApplicationStatus shouldBe
+ FinalApplicationStatus.KILLED
+ }
+ }
+
+ test("killing YARN app should change batch state to dead") {
+ val output = newOutputPath()
+ withTestLib(classOf[SimpleSparkApp], List(output, "false")) { s =>
+ s.verifySessionState(SessionState.Running())
+ val appId = s.appId()
+
+ // Kill the YARN app and check batch state should be KILLED.
+ cluster.yarnClient.killApplication(appId)
+ s.verifySessionDead()
+
+ cluster.yarnClient.getApplicationReport(appId).getFinalApplicationStatus shouldBe
+ FinalApplicationStatus.KILLED
+
+ s.snapshot().log should contain ("Application killed by user.")
+ }
+ }
+
+ test("recover batch sessions") {
+ val output1 = newOutputPath()
+ val output2 = newOutputPath()
+ withTestLib(classOf[SimpleSparkApp], List(output1)) { s1 =>
+ s1.stop()
+ withTestLib(classOf[SimpleSparkApp], List(output2, "false")) { s2 =>
+ s2.verifySessionRunning()
+
+ restartLivy()
+
+ // Verify previous active session still appears after restart.
+ s2.verifySessionRunning()
+ // Verify deleted session doesn't show up.
+ s1.verifySessionDoesNotExist()
+
+ s2.stop()
+
+ // Verify new session doesn't reuse old session id.
+ withTestLib(classOf[SimpleSparkApp], List(output2)) { s3 =>
+ s3.id should be > s2.id
+ }
+ }
+ }
+ }
+
+ private def newOutputPath(): String = {
+ cluster.hdfsScratchDir().toString() + "/" + UUID.randomUUID().toString()
+ }
+
+ private def uploadResource(name: String): String = {
+ val hdfsPath = new Path(cluster.hdfsScratchDir(), UUID.randomUUID().toString + "-" + name)
+ val in = getClass.getResourceAsStream("/" + name)
+ val out = cluster.fs.create(hdfsPath)
+ try {
+ IOUtils.copy(in, out)
+ } finally {
+ in.close()
+ out.close()
+ }
+ hdfsPath.toUri().getPath()
+ }
+
+ private def withScript[R]
+ (scriptPath: String, args: List[String], sparkConf: Map[String, String] = Map.empty)
+ (f: (LivyRestClient#BatchSession) => R): R = {
+ val s = livyClient.startBatch(scriptPath, None, args, sparkConf)
+ withSession(s)(f)
+ }
+
+ private def withTestLib[R]
+ (testClass: Class[_], args: List[String], sparkConf: Map[String, String] = Map.empty)
+ (f: (LivyRestClient#BatchSession) => R): R = {
+ val s = livyClient.startBatch(testLibPath, Some(testClass.getName()), args, sparkConf)
+ withSession(s)(f)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala b/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
new file mode 100644
index 0000000..728fd1c
--- /dev/null
+++ b/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test
+
+import java.util.concurrent.atomic.AtomicInteger
+import java.util.regex.Pattern
+
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.OptionValues._
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.apache.livy.rsc.RSCConf
+import org.apache.livy.sessions._
+import org.apache.livy.test.framework.{BaseIntegrationTestSuite, LivyRestClient}
+
+class InteractiveIT extends BaseIntegrationTestSuite {
+ test("basic interactive session") {
+ withNewSession(Spark()) { s =>
+ s.run("val sparkVersion = sc.version").result().left.foreach(info(_))
+ s.run("1+1").verifyResult("res0: Int = 2")
+ s.run("""sc.getConf.get("spark.executor.instances")""").verifyResult("res1: String = 1")
+ s.run("val sql = new org.apache.spark.sql.SQLContext(sc)").verifyResult(
+ ".*" + Pattern.quote(
+ "sql: org.apache.spark.sql.SQLContext = org.apache.spark.sql.SQLContext") + ".*")
+ s.run("abcde").verifyError(evalue = ".*?:[0-9]+: error: not found: value abcde.*")
+ s.run("throw new IllegalStateException()")
+ .verifyError(evalue = ".*java\\.lang\\.IllegalStateException.*")
+
+ // Verify Livy internal configurations are not exposed.
+ // TODO separate all these checks to different sub tests after merging new IT code.
+ s.run("""sc.getConf.getAll.exists(_._1.startsWith("spark.__livy__."))""")
+ .verifyResult(".*false")
+ s.run("""sys.props.exists(_._1.startsWith("spark.__livy__."))""").verifyResult(".*false")
+
+ // Make sure appInfo is reported correctly.
+ val state = s.snapshot()
+ state.appInfo.driverLogUrl.value should include ("containerlogs")
+ state.appInfo.sparkUiUrl.value should startWith ("http")
+
+ // Stop session and verify the YARN app state is finished.
+ // This is important because if YARN app state is killed, Spark history is not archived.
+ val appId = s.appId()
+ s.stop()
+ val appReport = cluster.yarnClient.getApplicationReport(appId)
+ appReport.getYarnApplicationState() shouldEqual YarnApplicationState.FINISHED
+ }
+ }
+
+ pytest("pyspark interactive session") {
+ withNewSession(PySpark()) { s =>
+ s.run("1+1").verifyResult("2")
+ s.run("sqlContext").verifyResult(startsWith("<pyspark.sql.context.HiveContext"))
+ s.run("sc.parallelize(range(100)).map(lambda x: x * 2).reduce(lambda x, y: x + y)")
+ .verifyResult("9900")
+ s.run("from pyspark.sql.types import Row").verifyResult("")
+ s.run("x = [Row(age=1, name=u'a'), Row(age=2, name=u'b'), Row(age=3, name=u'c')]")
+ .verifyResult("")
+ s.run("%table x").verifyResult(".*headers.*type.*name.*data.*")
+ s.run("abcde").verifyError(ename = "NameError", evalue = "name 'abcde' is not defined")
+ s.run("raise KeyError, 'foo'").verifyError(ename = "KeyError", evalue = "'foo'")
+ }
+ }
+
+ rtest("R interactive session") {
+ withNewSession(SparkR()) { s =>
+ // R's output sometimes includes the count of statements, which makes it annoying to test
+ // things. This helps a bit.
+ val curr = new AtomicInteger()
+ def count: Int = curr.incrementAndGet()
+
+ s.run("1+1").verifyResult(startsWith(s"[$count] 2"))
+ s.run("sqlContext <- sparkRSQL.init(sc)").verifyResult(null)
+ s.run("hiveContext <- sparkRHive.init(sc)").verifyResult(null)
+ s.run("""localDF <- data.frame(name=c("John", "Smith", "Sarah"), age=c(19, 23, 18))""")
+ .verifyResult(null)
+ s.run("df <- createDataFrame(sqlContext, localDF)").verifyResult(null)
+ s.run("printSchema(df)").verifyResult(literal(
+ """|root
+ | |-- name: string (nullable = true)
+ | |-- age: double (nullable = true)""".stripMargin))
+ }
+ }
+
+ test("application kills session") {
+ withNewSession(Spark()) { s =>
+ s.runFatalStatement("System.exit(0)")
+ }
+ }
+
+ test("should kill RSCDriver if it doesn't respond to end session") {
+ val testConfName = s"${RSCConf.LIVY_SPARK_PREFIX}${RSCConf.Entry.TEST_STUCK_END_SESSION.key()}"
+ withNewSession(Spark(), Map(testConfName -> "true")) { s =>
+ val appId = s.appId()
+ s.stop()
+ val appReport = cluster.yarnClient.getApplicationReport(appId)
+ appReport.getYarnApplicationState() shouldBe YarnApplicationState.KILLED
+ }
+ }
+
+ test("should kill RSCDriver if it didn't register itself in time") {
+ val testConfName =
+ s"${RSCConf.LIVY_SPARK_PREFIX}${RSCConf.Entry.TEST_STUCK_START_DRIVER.key()}"
+ withNewSession(Spark(), Map(testConfName -> "true"), false) { s =>
+ eventually(timeout(2 minutes), interval(5 seconds)) {
+ val appId = s.appId()
+ appId should not be null
+ val appReport = cluster.yarnClient.getApplicationReport(appId)
+ appReport.getYarnApplicationState() shouldBe YarnApplicationState.KILLED
+ }
+ }
+ }
+
+ test("user jars are properly imported in Scala interactive sessions") {
+ // Include a popular Java library to test importing user jars.
+ val sparkConf = Map("spark.jars.packages" -> "org.codehaus.plexus:plexus-utils:3.0.24")
+ withNewSession(Spark(), sparkConf) { s =>
+ // Check is the library loaded in JVM in the proper class loader.
+ s.run("Thread.currentThread.getContextClassLoader.loadClass" +
+ """("org.codehaus.plexus.util.FileUtils")""")
+ .verifyResult(".*Class\\[_\\] = class org.codehaus.plexus.util.FileUtils")
+
+ // Check does Scala interpreter see the library.
+ s.run("import org.codehaus.plexus.util._").verifyResult("import org.codehaus.plexus.util._")
+
+ // Check does SparkContext see classes defined by Scala interpreter.
+ s.run("case class Item(i: Int)").verifyResult("defined class Item")
+ s.run("val rdd = sc.parallelize(Array.fill(10){new Item(scala.util.Random.nextInt(1000))})")
+ .verifyResult("rdd.*")
+ s.run("rdd.count()").verifyResult(".*= 10")
+ }
+ }
+
+ test("heartbeat should kill expired session") {
+ // Set it to 2s because verifySessionIdle() is calling GET every second.
+ val heartbeatTimeout = Duration.create("2s")
+ withNewSession(Spark(), Map.empty, true, heartbeatTimeout.toSeconds.toInt) { s =>
+ // If the test reaches here, that means verifySessionIdle() is successfully keeping the
+ // session alive. Now verify heartbeat is killing expired session.
+ Thread.sleep(heartbeatTimeout.toMillis * 2)
+ s.verifySessionDoesNotExist()
+ }
+ }
+
+ test("recover interactive session") {
+ withNewSession(Spark()) { s =>
+ val stmt1 = s.run("1")
+ stmt1.verifyResult("res0: Int = 1")
+
+ restartLivy()
+
+ // Verify session still exists.
+ s.verifySessionIdle()
+ s.run("2").verifyResult("res1: Int = 2")
+ // Verify statement result is preserved.
+ stmt1.verifyResult("res0: Int = 1")
+
+ s.stop()
+
+ restartLivy()
+
+ // Verify deleted session doesn't show up after recovery.
+ s.verifySessionDoesNotExist()
+
+ // Verify new session doesn't reuse old session id.
+ withNewSession(Spark(), Map.empty, false) { s1 =>
+ s1.id should be > s.id
+ }
+ }
+ }
+
+ private def withNewSession[R] (
+ kind: Kind,
+ sparkConf: Map[String, String] = Map.empty,
+ waitForIdle: Boolean = true,
+ heartbeatTimeoutInSecond: Int = 0)
+ (f: (LivyRestClient#InteractiveSession) => R): R = {
+ withSession(livyClient.startSession(kind, sparkConf, heartbeatTimeoutInSecond)) { s =>
+ if (waitForIdle) {
+ s.verifySessionIdle()
+ }
+ f(s)
+ }
+ }
+
+ private def startsWith(result: String): String = Pattern.quote(result) + ".*"
+
+ private def literal(result: String): String = Pattern.quote(result)
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala b/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
new file mode 100644
index 0000000..becaaac
--- /dev/null
+++ b/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test
+
+import java.io.{File, InputStream}
+import java.net.URI
+import java.nio.charset.StandardCharsets.UTF_8
+import java.nio.file.{Files, StandardCopyOption}
+import java.util.concurrent.{Future => JFuture, TimeUnit}
+import javax.servlet.http.HttpServletResponse
+
+import scala.collection.JavaConverters._
+import scala.util.{Properties, Try}
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.livy._
+import org.apache.livy.client.common.HttpMessages._
+import org.apache.livy.sessions.SessionKindModule
+import org.apache.livy.test.framework.BaseIntegrationTestSuite
+import org.apache.livy.test.jobs._
+import org.apache.livy.utils.LivySparkUtils
+
+// Proper type representing the return value of "GET /sessions". At some point we should make
+// SessionServlet use something like this.
+class SessionList {
+ val from: Int = -1
+ val total: Int = -1
+ val sessions: List[SessionInfo] = Nil
+}
+
+class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logging {
+
+ private var client: LivyClient = _
+ private var sessionId: Int = _
+ private var client2: LivyClient = _
+ private val mapper = new ObjectMapper()
+ .registerModule(DefaultScalaModule)
+ .registerModule(new SessionKindModule())
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ Seq(client, client2).foreach { c =>
+ if (c != null) {
+ c.stop(true)
+ }
+ }
+
+ livyClient.connectSession(sessionId).stop()
+ }
+
+ test("create a new session and upload test jar") {
+ val tempClient = createClient(livyEndpoint)
+
+ try {
+ // Figure out the session ID by poking at the REST endpoint. We should probably expose this
+ // in the Java API.
+ val list = sessionList()
+ assert(list.total === 1)
+ val tempSessionId = list.sessions(0).id
+
+ livyClient.connectSession(tempSessionId).verifySessionIdle()
+ waitFor(tempClient.uploadJar(new File(testLib)))
+
+ client = tempClient
+ sessionId = tempSessionId
+ } finally {
+ if (client == null) {
+ try {
+ tempClient.stop(true)
+ } catch {
+ case e: Exception => warn("Error stopping client.", e)
+ }
+ }
+ }
+ }
+
+ test("upload file") {
+ assume(client != null, "Client not active.")
+
+ val file = Files.createTempFile("filetest", ".txt")
+ Files.write(file, "hello".getBytes(UTF_8))
+
+ waitFor(client.uploadFile(file.toFile()))
+
+ val result = waitFor(client.submit(new FileReader(file.toFile().getName(), false)))
+ assert(result === "hello")
+ }
+
+ test("add file from HDFS") {
+ assume(client != null, "Client not active.")
+ val file = Files.createTempFile("filetest2", ".txt")
+ Files.write(file, "hello".getBytes(UTF_8))
+
+ val uri = new URI(uploadToHdfs(file.toFile()))
+ waitFor(client.addFile(uri))
+
+ val task = new FileReader(new File(uri.getPath()).getName(), false)
+ val result = waitFor(client.submit(task))
+ assert(result === "hello")
+ }
+
+ test("run simple jobs") {
+ assume(client != null, "Client not active.")
+
+ val result = waitFor(client.submit(new Echo("hello")))
+ assert(result === "hello")
+
+ val result2 = waitFor(client.run(new Echo("hello")))
+ assert(result2 === "hello")
+ }
+
+ test("run spark job") {
+ assume(client != null, "Client not active.")
+ val result = waitFor(client.submit(new SmallCount(100)))
+ assert(result === 100)
+ }
+
+ test("run spark sql job") {
+ assume(client != null, "Client not active.")
+ val result = waitFor(client.submit(new SQLGetTweets(false)))
+ assert(result.size() > 0)
+ }
+
+ test("stop a client without destroying the session") {
+ assume(client != null, "Client not active.")
+ client.stop(false)
+ client = null
+ }
+
+ test("connect to an existing session") {
+ livyClient.connectSession(sessionId).verifySessionIdle()
+ val sessionUri = s"$livyEndpoint/sessions/$sessionId"
+ client2 = createClient(sessionUri)
+ }
+
+ test("submit job using new client") {
+ assume(client2 != null, "Client not active.")
+ val result = waitFor(client2.submit(new Echo("hello")))
+ assert(result === "hello")
+ }
+
+ scalaTest("run scala jobs") {
+ assume(client2 != null, "Client not active.")
+
+ val jobs = Seq(
+ new ScalaEcho("abcde"),
+ new ScalaEcho(Seq(1, 2, 3, 4)),
+ new ScalaEcho(Map(1 -> 2, 3 -> 4)),
+ new ScalaEcho(ValueHolder("abcde")),
+ new ScalaEcho(ValueHolder(Seq(1, 2, 3, 4))),
+ new ScalaEcho(Some("abcde"))
+ )
+
+ jobs.foreach { job =>
+ val result = waitFor(client2.submit(job))
+ assert(result === job.value)
+ }
+ }
+
+ protected def scalaTest(desc: String)(testFn: => Unit): Unit = {
+ test(desc) {
+ assume(sys.env("LIVY_SPARK_SCALA_VERSION") ==
+ LivySparkUtils.formatScalaVersion(Properties.versionNumberString),
+ s"Scala test can only be run with ${Properties.versionString}")
+ testFn
+ }
+ }
+
+ test("ensure failing jobs do not affect session state") {
+ assume(client2 != null, "Client not active.")
+
+ try {
+ waitFor(client2.submit(new Failure()))
+ fail("Job should have failued.")
+ } catch {
+ case e: Exception =>
+ assert(e.getMessage().contains(classOf[Failure.JobFailureException].getName()))
+ }
+
+ val result = waitFor(client2.submit(new Echo("foo")))
+ assert(result === "foo")
+ }
+
+ test("return null should not throw NPE") {
+ assume(client2 != null, "Client not active")
+
+ val result = waitFor(client2.submit(new VoidJob()))
+ assert(result === null)
+ }
+
+ test("destroy the session") {
+ assume(client2 != null, "Client not active.")
+ client2.stop(true)
+
+ val list = sessionList()
+ assert(list.total === 0)
+
+ val sessionUri = s"$livyEndpoint/sessions/$sessionId"
+ Try(createClient(sessionUri)).foreach { client =>
+ client.stop(true)
+ fail("Should not have been able to connect to destroyed session.")
+ }
+
+ sessionId = -1
+ }
+
+ pytest("validate Python-API requests") {
+ val addFileContent = "hello from addfile"
+ val addFilePath = createTempFilesForTest("add_file", ".txt", addFileContent, true)
+ val addPyFileContent = "def test_add_pyfile(): return \"hello from addpyfile\""
+ val addPyFilePath = createTempFilesForTest("add_pyfile", ".py", addPyFileContent, true)
+ val uploadFileContent = "hello from uploadfile"
+ val uploadFilePath = createTempFilesForTest("upload_pyfile", ".py", uploadFileContent, false)
+ val uploadPyFileContent = "def test_upload_pyfile(): return \"hello from uploadpyfile\""
+ val uploadPyFilePath = createTempFilesForTest("upload_pyfile", ".py",
+ uploadPyFileContent, false)
+
+ val builder = new ProcessBuilder(Seq("python", createPyTestsForPythonAPI().toString).asJava)
+
+ val env = builder.environment()
+ env.put("LIVY_END_POINT", livyEndpoint)
+ env.put("ADD_FILE_URL", addFilePath)
+ env.put("ADD_PYFILE_URL", addPyFilePath)
+ env.put("UPLOAD_FILE_URL", uploadFilePath)
+ env.put("UPLOAD_PYFILE_URL", uploadPyFilePath)
+
+ builder.redirectOutput(new File(sys.props("java.io.tmpdir") + "/pytest_results.txt"))
+ builder.redirectErrorStream(true)
+
+ val process = builder.start()
+
+ process.waitFor()
+
+ assert(process.exitValue() === 0)
+ }
+
+ private def createPyTestsForPythonAPI(): File = {
+ var source: InputStream = null
+ try {
+ source = getClass.getClassLoader.getResourceAsStream("test_python_api.py")
+ val file = Files.createTempFile("", "").toFile
+ Files.copy(source, file.toPath, StandardCopyOption.REPLACE_EXISTING)
+ file
+ } finally {
+ source.close()
+ }
+ }
+
+ private def createTempFilesForTest(
+ fileName: String,
+ fileExtension: String,
+ fileContent: String,
+ uploadFileToHdfs: Boolean): String = {
+ val path = Files.createTempFile(fileName, fileExtension)
+ Files.write(path, fileContent.getBytes(UTF_8))
+ if (uploadFileToHdfs) {
+ uploadToHdfs(path.toFile())
+ } else {
+ path.toString
+ }
+ }
+
+ private def waitFor[T](future: JFuture[T]): T = {
+ future.get(30, TimeUnit.SECONDS)
+ }
+
+ private def sessionList(): SessionList = {
+ val response = httpClient.prepareGet(s"$livyEndpoint/sessions/").execute().get()
+ assert(response.getStatusCode === HttpServletResponse.SC_OK)
+ mapper.readValue(response.getResponseBodyAsStream, classOf[SessionList])
+ }
+
+ private def createClient(uri: String): LivyClient = {
+ new LivyClientBuilder().setURI(new URI(uri)).build()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/test/spark2/scala/Spark2JobApiIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/spark2/scala/Spark2JobApiIT.scala b/integration-test/src/test/spark2/scala/Spark2JobApiIT.scala
index 0501685..aa30b88 100644
--- a/integration-test/src/test/spark2/scala/Spark2JobApiIT.scala
+++ b/integration-test/src/test/spark2/scala/Spark2JobApiIT.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package com.cloudera.livy.test
+package org.apache.livy.test
import java.io.File
import java.net.URI
@@ -26,11 +26,11 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.scalatest.BeforeAndAfterAll
-import com.cloudera.livy._
-import com.cloudera.livy.client.common.HttpMessages._
-import com.cloudera.livy.sessions.SessionKindModule
-import com.cloudera.livy.test.framework.BaseIntegrationTestSuite
-import com.cloudera.livy.test.jobs.spark2._
+import org.apache.livy._
+import org.apache.livy.client.common.HttpMessages._
+import org.apache.livy.sessions.SessionKindModule
+import org.apache.livy.test.framework.BaseIntegrationTestSuite
+import org.apache.livy.test.jobs.spark2._
class Spark2JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logging {
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d139099..9c1263b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -20,9 +20,9 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<name>livy-main</name>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/python-api/pom.xml
----------------------------------------------------------------------
diff --git a/python-api/pom.xml b/python-api/pom.xml
index 3ea50c1..86379ce 100644
--- a/python-api/pom.xml
+++ b/python-api/pom.xml
@@ -21,15 +21,15 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-python-api</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/python-api/setup.py
----------------------------------------------------------------------
diff --git a/python-api/setup.py b/python-api/setup.py
index 5f55ffd..a7e3aab 100644
--- a/python-api/setup.py
+++ b/python-api/setup.py
@@ -39,14 +39,14 @@ requirements = [
setup(
name='livy-python-api',
- version="0.4.0-SNAPSHOT",
+ version="0.4.0-incubating-SNAPSHOT",
packages=["livy", "livy-tests"],
package_dir={
"": "src/main/python",
"livy-tests": "src/test/python/livy-tests",
},
- url='https://github.com/cloudera/livy',
- author_email='livy-user@cloudera.org',
+ url='https://github.com/apache/incubator-livy',
+ author_email='user@livy.incubator.apache.org',
license='Apache License, Version 2.0',
description=DESCRIPTION,
platforms=['any'],
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/pom.xml
----------------------------------------------------------------------
diff --git a/repl/pom.xml b/repl/pom.xml
index 3af51bd..14931ba 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -20,33 +20,33 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>multi-scala-project-root</artifactId>
<relativePath>../scala/pom.xml</relativePath>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>livy-repl-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<dependencies>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-rsc</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-core_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<exclusions>
<!-- Provided and shaded by livy-rsc already. -->
<exclusion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-client-common</artifactId>
</exclusion>
</exclusions>
@@ -189,7 +189,7 @@
<relocations>
<relocation>
<pattern>org.json4s</pattern>
- <shadedPattern>com.cloudera.livy.shaded.json4s</shadedPattern>
+ <shadedPattern>org.apache.livy.shaded.json4s</shadedPattern>
</relocation>
</relocations>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/scala-2.10/pom.xml
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/pom.xml b/repl/scala-2.10/pom.xml
index c81659b..4d84611 100644
--- a/repl/scala-2.10/pom.xml
+++ b/repl/scala-2.10/pom.xml
@@ -19,15 +19,15 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-repl_2.10</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-repl-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala b/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
deleted file mode 100644
index dc761cc..0000000
--- a/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import java.io._
-import java.net.URLClassLoader
-import java.nio.file.Paths
-
-import scala.tools.nsc.Settings
-import scala.tools.nsc.interpreter.JPrintWriter
-import scala.tools.nsc.interpreter.Results.Result
-import scala.util.{Failure, Success, Try}
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.repl.SparkIMain
-
-/**
- * This represents a Spark interpreter. It is not thread safe.
- */
-class SparkInterpreter(conf: SparkConf)
- extends AbstractSparkInterpreter with SparkContextInitializer {
-
- private var sparkIMain: SparkIMain = _
- protected var sparkContext: SparkContext = _
-
- override def start(): SparkContext = {
- require(sparkIMain == null && sparkContext == null)
-
- val settings = new Settings()
- settings.embeddedDefaults(Thread.currentThread().getContextClassLoader())
- settings.usejavacp.value = true
-
- sparkIMain = new SparkIMain(settings, new JPrintWriter(outputStream, true))
- sparkIMain.initializeSynchronous()
-
- // Spark 1.6 does not have "classServerUri"; instead, the local directory where class files
- // are stored needs to be registered in SparkConf. See comment in
- // SparkILoop::createSparkContext().
- Try(sparkIMain.getClass().getMethod("classServerUri")) match {
- case Success(method) =>
- method.setAccessible(true)
- conf.set("spark.repl.class.uri", method.invoke(sparkIMain).asInstanceOf[String])
-
- case Failure(_) =>
- val outputDir = sparkIMain.getClass().getMethod("getClassOutputDirectory")
- outputDir.setAccessible(true)
- conf.set("spark.repl.class.outputDir",
- outputDir.invoke(sparkIMain).asInstanceOf[File].getAbsolutePath())
- }
-
- restoreContextClassLoader {
- // Call sparkIMain.setContextClassLoader() to make sure SparkContext and repl are using the
- // same ClassLoader. Otherwise if someone defined a new class in interactive shell,
- // SparkContext cannot see them and will result in job stage failure.
- val setContextClassLoaderMethod = sparkIMain.getClass().getMethod("setContextClassLoader")
- setContextClassLoaderMethod.setAccessible(true)
- setContextClassLoaderMethod.invoke(sparkIMain)
-
- // With usejavacp=true, the Scala interpreter looks for jars under System Classpath. But it
- // doesn't look for jars added to MutableURLClassLoader. Thus extra jars are not visible to
- // the interpreter. SparkContext can use them via JVM ClassLoaders but users cannot import
- // them using Scala import statement.
- //
- // For instance: If we import a package using SparkConf:
- // "spark.jars.packages": "com.databricks:spark-csv_2.10:1.4.0"
- // then "import com.databricks.spark.csv._" in the interpreter, it will throw an error.
- //
- // Adding them to the interpreter manually to fix this issue.
- var classLoader = Thread.currentThread().getContextClassLoader
- while (classLoader != null) {
- if (classLoader.getClass.getCanonicalName == "org.apache.spark.util.MutableURLClassLoader")
- {
- val extraJarPath = classLoader.asInstanceOf[URLClassLoader].getURLs()
- // Check if the file exists. Otherwise an exception will be thrown.
- .filter { u => u.getProtocol == "file" && new File(u.getPath).isFile }
- // Livy rsc and repl are also in the extra jars list. Filter them out.
- .filterNot { u => Paths.get(u.toURI).getFileName.toString.startsWith("livy-") }
- // Some bad spark packages depend on the wrong version of scala-reflect. Blacklist it.
- .filterNot { u =>
- Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect")
- }
-
- extraJarPath.foreach { p => debug(s"Adding $p to Scala interpreter's class path...") }
- sparkIMain.addUrlsToClassPath(extraJarPath: _*)
- classLoader = null
- } else {
- classLoader = classLoader.getParent
- }
- }
-
- createSparkContext(conf)
- }
-
- sparkContext
- }
-
- protected def bind(name: String, tpe: String, value: Object, modifier: List[String]): Unit = {
- sparkIMain.beQuietDuring {
- sparkIMain.bind(name, tpe, value, modifier)
- }
- }
-
- override def close(): Unit = synchronized {
- if (sparkContext != null) {
- sparkContext.stop()
- sparkContext = null
- }
-
- if (sparkIMain != null) {
- sparkIMain.close()
- sparkIMain = null
- }
- }
-
- override protected def isStarted(): Boolean = {
- sparkContext != null && sparkIMain != null
- }
-
- override protected def interpret(code: String): Result = {
- sparkIMain.interpret(code)
- }
-
- override protected[repl] def parseError(stdout: String): (String, Seq[String]) = {
- // An example of Scala 2.10 runtime exception error message:
- // java.lang.Exception: message
- // at $iwC$$iwC$$iwC$$iwC$$iwC.error(<console>:25)
- // at $iwC$$iwC$$iwC.error2(<console>:27)
- // at $iwC$$iwC.<init>(<console>:41)
- // at $iwC.<init>(<console>:43)
- // at <init>(<console>:45)
- // at .<init>(<console>:49)
- // at .<clinit>(<console>)
- // at .<init>(<console>:7)
- // at .<clinit>(<console>)
- // at $print(<console>)
- // at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- // at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- // ...
-
- val (ename, traceback) = super.parseError(stdout)
-
- // Remove internal frames.
- val startOfInternalFrames = traceback.indexWhere(_.contains("$iwC$$iwC.<init>"))
- var endOfInternalFrames = traceback.indexWhere(!_.trim.startsWith("at"), startOfInternalFrames)
- if (endOfInternalFrames == -1) {
- endOfInternalFrames = traceback.length
- }
-
- val cleanedTraceback = if (startOfInternalFrames == -1) {
- traceback
- } else {
- traceback.view.zipWithIndex
- .filterNot { z => z._2 >= startOfInternalFrames && z._2 < endOfInternalFrames }
- .map { _._1.replaceAll("(\\$iwC\\$)*\\$iwC", "<user code>") }
- }
-
- (ename, cleanedTraceback)
- }
-
- override protected def valueOfTerm(name: String): Option[Any] = {
- sparkIMain.valueOfTerm(name)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/scala-2.10/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala b/repl/scala-2.10/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
new file mode 100644
index 0000000..f5b5b32
--- /dev/null
+++ b/repl/scala-2.10/src/main/scala/org/apache/livy/repl/SparkInterpreter.scala
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.repl
+
+import java.io._
+import java.net.URLClassLoader
+import java.nio.file.Paths
+
+import scala.tools.nsc.Settings
+import scala.tools.nsc.interpreter.JPrintWriter
+import scala.tools.nsc.interpreter.Results.Result
+import scala.util.{Failure, Success, Try}
+
+import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.repl.SparkIMain
+
+/**
+ * This represents a Spark interpreter. It is not thread safe.
+ */
+class SparkInterpreter(conf: SparkConf)
+ extends AbstractSparkInterpreter with SparkContextInitializer {
+
+ private var sparkIMain: SparkIMain = _
+ protected var sparkContext: SparkContext = _
+
+ override def start(): SparkContext = {
+ require(sparkIMain == null && sparkContext == null)
+
+ val settings = new Settings()
+ settings.embeddedDefaults(Thread.currentThread().getContextClassLoader())
+ settings.usejavacp.value = true
+
+ sparkIMain = new SparkIMain(settings, new JPrintWriter(outputStream, true))
+ sparkIMain.initializeSynchronous()
+
+ // Spark 1.6 does not have "classServerUri"; instead, the local directory where class files
+ // are stored needs to be registered in SparkConf. See comment in
+ // SparkILoop::createSparkContext().
+ Try(sparkIMain.getClass().getMethod("classServerUri")) match {
+ case Success(method) =>
+ method.setAccessible(true)
+ conf.set("spark.repl.class.uri", method.invoke(sparkIMain).asInstanceOf[String])
+
+ case Failure(_) =>
+ val outputDir = sparkIMain.getClass().getMethod("getClassOutputDirectory")
+ outputDir.setAccessible(true)
+ conf.set("spark.repl.class.outputDir",
+ outputDir.invoke(sparkIMain).asInstanceOf[File].getAbsolutePath())
+ }
+
+ restoreContextClassLoader {
+ // Call sparkIMain.setContextClassLoader() to make sure SparkContext and repl are using the
+ // same ClassLoader. Otherwise if someone defined a new class in interactive shell,
+ // SparkContext cannot see them and will result in job stage failure.
+ val setContextClassLoaderMethod = sparkIMain.getClass().getMethod("setContextClassLoader")
+ setContextClassLoaderMethod.setAccessible(true)
+ setContextClassLoaderMethod.invoke(sparkIMain)
+
+ // With usejavacp=true, the Scala interpreter looks for jars under System Classpath. But it
+ // doesn't look for jars added to MutableURLClassLoader. Thus extra jars are not visible to
+ // the interpreter. SparkContext can use them via JVM ClassLoaders but users cannot import
+ // them using Scala import statement.
+ //
+ // For instance: If we import a package using SparkConf:
+ // "spark.jars.packages": "com.databricks:spark-csv_2.10:1.4.0"
+ // then "import com.databricks.spark.csv._" in the interpreter, it will throw an error.
+ //
+ // Adding them to the interpreter manually to fix this issue.
+ var classLoader = Thread.currentThread().getContextClassLoader
+ while (classLoader != null) {
+ if (classLoader.getClass.getCanonicalName == "org.apache.spark.util.MutableURLClassLoader")
+ {
+ val extraJarPath = classLoader.asInstanceOf[URLClassLoader].getURLs()
+ // Check if the file exists. Otherwise an exception will be thrown.
+ .filter { u => u.getProtocol == "file" && new File(u.getPath).isFile }
+ // Livy rsc and repl are also in the extra jars list. Filter them out.
+ .filterNot { u => Paths.get(u.toURI).getFileName.toString.startsWith("livy-") }
+ // Some bad spark packages depend on the wrong version of scala-reflect. Blacklist it.
+ .filterNot { u =>
+ Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect")
+ }
+
+ extraJarPath.foreach { p => debug(s"Adding $p to Scala interpreter's class path...") }
+ sparkIMain.addUrlsToClassPath(extraJarPath: _*)
+ classLoader = null
+ } else {
+ classLoader = classLoader.getParent
+ }
+ }
+
+ createSparkContext(conf)
+ }
+
+ sparkContext
+ }
+
+ protected def bind(name: String, tpe: String, value: Object, modifier: List[String]): Unit = {
+ sparkIMain.beQuietDuring {
+ sparkIMain.bind(name, tpe, value, modifier)
+ }
+ }
+
+ override def close(): Unit = synchronized {
+ if (sparkContext != null) {
+ sparkContext.stop()
+ sparkContext = null
+ }
+
+ if (sparkIMain != null) {
+ sparkIMain.close()
+ sparkIMain = null
+ }
+ }
+
+ override protected def isStarted(): Boolean = {
+ sparkContext != null && sparkIMain != null
+ }
+
+ override protected def interpret(code: String): Result = {
+ sparkIMain.interpret(code)
+ }
+
+ override protected[repl] def parseError(stdout: String): (String, Seq[String]) = {
+ // An example of Scala 2.10 runtime exception error message:
+ // java.lang.Exception: message
+ // at $iwC$$iwC$$iwC$$iwC$$iwC.error(<console>:25)
+ // at $iwC$$iwC$$iwC.error2(<console>:27)
+ // at $iwC$$iwC.<init>(<console>:41)
+ // at $iwC.<init>(<console>:43)
+ // at <init>(<console>:45)
+ // at .<init>(<console>:49)
+ // at .<clinit>(<console>)
+ // at .<init>(<console>:7)
+ // at .<clinit>(<console>)
+ // at $print(<console>)
+ // at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+ // at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
+ // ...
+
+ val (ename, traceback) = super.parseError(stdout)
+
+ // Remove internal frames.
+ val startOfInternalFrames = traceback.indexWhere(_.contains("$iwC$$iwC.<init>"))
+ var endOfInternalFrames = traceback.indexWhere(!_.trim.startsWith("at"), startOfInternalFrames)
+ if (endOfInternalFrames == -1) {
+ endOfInternalFrames = traceback.length
+ }
+
+ val cleanedTraceback = if (startOfInternalFrames == -1) {
+ traceback
+ } else {
+ traceback.view.zipWithIndex
+ .filterNot { z => z._2 >= startOfInternalFrames && z._2 < endOfInternalFrames }
+ .map { _._1.replaceAll("(\\$iwC\\$)*\\$iwC", "<user code>") }
+ }
+
+ (ename, cleanedTraceback)
+ }
+
+ override protected def valueOfTerm(name: String): Option[Any] = {
+ sparkIMain.valueOfTerm(name)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/repl/scala-2.10/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala b/repl/scala-2.10/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
deleted file mode 100644
index 960b59e..0000000
--- a/repl/scala-2.10/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.repl
-
-import org.scalatest._
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-
-class SparkInterpreterSpec extends FunSpec with Matchers with LivyBaseUnitTestSuite {
- describe("SparkInterpreter") {
- val interpreter = new SparkInterpreter(null)
-
- it("should parse Scala compile error.") {
- // Regression test for LIVY-260.
- val error =
- """<console>:27: error: type mismatch;
- | found : Int
- | required: String
- | sc.setJobGroup(groupName, groupName, true)
- | ^
- |<console>:27: error: type mismatch;
- | found : Int
- | required: String
- | sc.setJobGroup(groupName, groupName, true)
- | ^
- |""".stripMargin
-
- val expectedTraceback = AbstractSparkInterpreter.KEEP_NEWLINE_REGEX.split(
- """ found : Int
- | required: String
- | sc.setJobGroup(groupName, groupName, true)
- | ^
- |<console>:27: error: type mismatch;
- | found : Int
- | required: String
- | sc.setJobGroup(groupName, groupName, true)
- | ^
- |""".stripMargin)
-
- val (ename, traceback) = interpreter.parseError(error)
- ename shouldBe "<console>:27: error: type mismatch;"
- traceback shouldBe expectedTraceback
- }
-
- it("should parse Scala runtime error and remove internal frames.") {
- val error =
- """java.lang.RuntimeException: message
- | at $iwC$$iwC$$iwC$$iwC$$iwC.error(<console>:25)
- | at $iwC$$iwC$$iwC.error2(<console>:27)
- | at $iwC$$iwC.<init>(<console>:41)
- | at $iwC.<init>(<console>:43)
- | at <init>(<console>:45)
- | at .<init>(<console>:49)
- | at .<clinit>(<console>)
- | at .<init>(<console>:7)
- | at .<clinit>(<console>)
- | at $print(<console>)
- | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- |""".stripMargin
-
- val expectedTraceback = AbstractSparkInterpreter.KEEP_NEWLINE_REGEX.split(
- """ at <user code>.error(<console>:25)
- | at <user code>.error2(<console>:27)
- |""".stripMargin)
-
- val (ename, traceback) = interpreter.parseError(error)
- ename shouldBe "java.lang.RuntimeException: message"
- traceback shouldBe expectedTraceback
- }
- }
-}
[05/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/sessions/SessionManager.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/sessions/SessionManager.scala b/server/src/main/scala/org/apache/livy/sessions/SessionManager.scala
new file mode 100644
index 0000000..d482c33
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/sessions/SessionManager.scala
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.sessions
+
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicInteger
+
+import scala.collection.mutable
+import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.concurrent.duration.Duration
+import scala.reflect.ClassTag
+import scala.util.control.NonFatal
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.server.batch.{BatchRecoveryMetadata, BatchSession}
+import org.apache.livy.server.interactive.{InteractiveRecoveryMetadata, InteractiveSession, SessionHeartbeatWatchdog}
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.Session.RecoveryMetadata
+
+object SessionManager {
+ val SESSION_RECOVERY_MODE_OFF = "off"
+ val SESSION_RECOVERY_MODE_RECOVERY = "recovery"
+}
+
+class BatchSessionManager(
+ livyConf: LivyConf,
+ sessionStore: SessionStore,
+ mockSessions: Option[Seq[BatchSession]] = None)
+ extends SessionManager[BatchSession, BatchRecoveryMetadata] (
+ livyConf, BatchSession.recover(_, livyConf, sessionStore), sessionStore, "batch", mockSessions)
+
+class InteractiveSessionManager(
+ livyConf: LivyConf,
+ sessionStore: SessionStore,
+ mockSessions: Option[Seq[InteractiveSession]] = None)
+ extends SessionManager[InteractiveSession, InteractiveRecoveryMetadata] (
+ livyConf,
+ InteractiveSession.recover(_, livyConf, sessionStore),
+ sessionStore,
+ "interactive",
+ mockSessions)
+ with SessionHeartbeatWatchdog[InteractiveSession, InteractiveRecoveryMetadata]
+ {
+ start()
+ }
+
+class SessionManager[S <: Session, R <: RecoveryMetadata : ClassTag](
+ protected val livyConf: LivyConf,
+ sessionRecovery: R => S,
+ sessionStore: SessionStore,
+ sessionType: String,
+ mockSessions: Option[Seq[S]] = None)
+ extends Logging {
+
+ import SessionManager._
+
+ protected implicit def executor: ExecutionContext = ExecutionContext.global
+
+ protected[this] final val idCounter = new AtomicInteger(0)
+ protected[this] final val sessions = mutable.LinkedHashMap[Int, S]()
+
+ private[this] final val sessionTimeoutCheck = livyConf.getBoolean(LivyConf.SESSION_TIMEOUT_CHECK)
+ private[this] final val sessionTimeout =
+ TimeUnit.MILLISECONDS.toNanos(livyConf.getTimeAsMs(LivyConf.SESSION_TIMEOUT))
+ private[this] final val sessionStateRetainedInSec =
+ TimeUnit.MILLISECONDS.toNanos(livyConf.getTimeAsMs(LivyConf.SESSION_STATE_RETAIN_TIME))
+
+ mockSessions.getOrElse(recover()).foreach(register)
+ new GarbageCollector().start()
+
+ def nextId(): Int = synchronized {
+ val id = idCounter.getAndIncrement()
+ sessionStore.saveNextSessionId(sessionType, idCounter.get())
+ id
+ }
+
+ def register(session: S): S = {
+ info(s"Registering new session ${session.id}")
+ synchronized {
+ sessions.put(session.id, session)
+ }
+ session
+ }
+
+ def get(id: Int): Option[S] = sessions.get(id)
+
+ def size(): Int = sessions.size
+
+ def all(): Iterable[S] = sessions.values
+
+ def delete(id: Int): Option[Future[Unit]] = {
+ get(id).map(delete)
+ }
+
+ def delete(session: S): Future[Unit] = {
+ session.stop().map { case _ =>
+ try {
+ sessionStore.remove(sessionType, session.id)
+ synchronized {
+ sessions.remove(session.id)
+ }
+ } catch {
+ case NonFatal(e) =>
+ error("Exception was thrown during stop session:", e)
+ throw e
+ }
+ }
+ }
+
+ def shutdown(): Unit = {
+ val recoveryEnabled = livyConf.get(LivyConf.RECOVERY_MODE) != SESSION_RECOVERY_MODE_OFF
+ if (!recoveryEnabled) {
+ sessions.values.map(_.stop).foreach { future =>
+ Await.ready(future, Duration.Inf)
+ }
+ }
+ }
+
+ def collectGarbage(): Future[Iterable[Unit]] = {
+ def expired(session: Session): Boolean = {
+ session.state match {
+ case s: FinishedSessionState =>
+ val currentTime = System.nanoTime()
+ currentTime - s.time > sessionStateRetainedInSec
+ case _ =>
+ if (!sessionTimeoutCheck) {
+ false
+ } else if (session.isInstanceOf[BatchSession]) {
+ false
+ } else {
+ val currentTime = System.nanoTime()
+ currentTime - session.lastActivity > sessionTimeout
+ }
+ }
+ }
+
+ Future.sequence(all().filter(expired).map(delete))
+ }
+
+ private def recover(): Seq[S] = {
+ // Recover next session id from state store and create SessionManager.
+ idCounter.set(sessionStore.getNextSessionId(sessionType))
+
+ // Retrieve session recovery metadata from state store.
+ val sessionMetadata = sessionStore.getAllSessions[R](sessionType)
+
+ // Recover session from session recovery metadata.
+ val recoveredSessions = sessionMetadata.flatMap(_.toOption).map(sessionRecovery)
+
+ info(s"Recovered ${recoveredSessions.length} $sessionType sessions." +
+ s" Next session id: $idCounter")
+
+ // Print recovery error.
+ val recoveryFailure = sessionMetadata.filter(_.isFailure).map(_.failed.get)
+ recoveryFailure.foreach(ex => error(ex.getMessage, ex.getCause))
+
+ recoveredSessions
+ }
+
+ private class GarbageCollector extends Thread("session gc thread") {
+
+ setDaemon(true)
+
+ override def run(): Unit = {
+ while (true) {
+ collectGarbage()
+ Thread.sleep(60 * 1000)
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/utils/Clock.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/utils/Clock.scala b/server/src/main/scala/org/apache/livy/utils/Clock.scala
new file mode 100644
index 0000000..8b396c7
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/utils/Clock.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.utils
+
+/**
+ * A lot of Livy code relies on time related functions like Thread.sleep.
+ * To timing effects from unit test, this class is created to mock out time.
+ *
+ * Code in Livy should not call Thread.sleep() directly. It should call this class instead.
+ */
+object Clock {
+ private var _sleep: Long => Unit = Thread.sleep
+
+ def withSleepMethod(mockSleep: Long => Unit)(f: => Unit): Unit = {
+ try {
+ _sleep = mockSleep
+ f
+ } finally {
+ _sleep = Thread.sleep
+ }
+ }
+
+ def sleep: Long => Unit = _sleep
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/utils/LineBufferedProcess.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/utils/LineBufferedProcess.scala b/server/src/main/scala/org/apache/livy/utils/LineBufferedProcess.scala
new file mode 100644
index 0000000..863f9a6
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/utils/LineBufferedProcess.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.utils
+
+import org.apache.livy.{Logging, Utils}
+
+class LineBufferedProcess(process: Process) extends Logging {
+
+ private[this] val _inputStream = new LineBufferedStream(process.getInputStream)
+ private[this] val _errorStream = new LineBufferedStream(process.getErrorStream)
+
+ def inputLines: IndexedSeq[String] = _inputStream.lines
+ def errorLines: IndexedSeq[String] = _errorStream.lines
+
+ def inputIterator: Iterator[String] = _inputStream.iterator
+ def errorIterator: Iterator[String] = _errorStream.iterator
+
+ def destroy(): Unit = {
+ process.destroy()
+ }
+
+ /** Returns if the process is still actively running. */
+ def isAlive: Boolean = Utils.isProcessAlive(process)
+
+ def exitValue(): Int = {
+ process.exitValue()
+ }
+
+ def waitFor(): Int = {
+ val returnCode = process.waitFor()
+ _inputStream.waitUntilClose()
+ _errorStream.waitUntilClose()
+ returnCode
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/utils/LineBufferedStream.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/utils/LineBufferedStream.scala b/server/src/main/scala/org/apache/livy/utils/LineBufferedStream.scala
new file mode 100644
index 0000000..fb076e1
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/utils/LineBufferedStream.scala
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.utils
+
+import java.io.InputStream
+import java.util.concurrent.locks.ReentrantLock
+
+import scala.io.Source
+
+import org.apache.livy.Logging
+
+class LineBufferedStream(inputStream: InputStream) extends Logging {
+
+ private[this] var _lines: IndexedSeq[String] = IndexedSeq()
+
+ private[this] val _lock = new ReentrantLock()
+ private[this] val _condition = _lock.newCondition()
+ private[this] var _finished = false
+
+ private val thread = new Thread {
+ override def run() = {
+ val lines = Source.fromInputStream(inputStream).getLines()
+ for (line <- lines) {
+ _lock.lock()
+ try {
+ _lines = _lines :+ line
+ _condition.signalAll()
+ } finally {
+ _lock.unlock()
+ }
+ }
+
+ _lines.map { line => info("stdout: ", line) }
+ _lock.lock()
+ try {
+ _finished = true
+ _condition.signalAll()
+ } finally {
+ _lock.unlock()
+ }
+ }
+ }
+ thread.setDaemon(true)
+ thread.start()
+
+ def lines: IndexedSeq[String] = _lines
+
+ def iterator: Iterator[String] = {
+ new LinesIterator
+ }
+
+ def waitUntilClose(): Unit = thread.join()
+
+ private class LinesIterator extends Iterator[String] {
+ private[this] var index = 0
+
+ override def hasNext: Boolean = {
+ if (index < _lines.length) {
+ true
+ } else {
+ // Otherwise we might still have more data.
+ _lock.lock()
+ try {
+ if (_finished) {
+ false
+ } else {
+ _condition.await()
+ index < _lines.length
+ }
+ } finally {
+ _lock.unlock()
+ }
+ }
+ }
+
+ override def next(): String = {
+ val line = _lines(index)
+ index += 1
+ line
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala b/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala
new file mode 100644
index 0000000..df83ca6
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.utils
+
+import java.io.{File, IOException}
+
+import scala.collection.SortedMap
+import scala.math.Ordering.Implicits._
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.LivyConf.LIVY_SPARK_SCALA_VERSION
+
+object LivySparkUtils extends Logging {
+
+ // For each Spark version we supported, we need to add this mapping relation in case Scala
+ // version cannot be detected from "spark-submit --version".
+ private val _defaultSparkScalaVersion = SortedMap(
+ // Spark 2.1 + Scala 2.11
+ (2, 1) -> "2.11",
+ // Spark 2.0 + Scala 2.11
+ (2, 0) -> "2.11",
+ // Spark 1.6 + Scala 2.10
+ (1, 6) -> "2.10"
+ )
+
+ // Supported Spark version
+ private val MIN_VERSION = (1, 6)
+ private val MAX_VERSION = (2, 2)
+
+ private val sparkVersionRegex = """version (.*)""".r.unanchored
+ private val scalaVersionRegex = """Scala version (.*), Java""".r.unanchored
+
+ /**
+ * Test that Spark home is configured and configured Spark home is a directory.
+ */
+ def testSparkHome(livyConf: LivyConf): Unit = {
+ val sparkHome = livyConf.sparkHome().getOrElse {
+ throw new IllegalArgumentException("Livy requires the SPARK_HOME environment variable")
+ }
+
+ require(new File(sparkHome).isDirectory(), "SPARK_HOME path does not exist")
+ }
+
+ /**
+ * Test that the configured `spark-submit` executable exists.
+ *
+ * @param livyConf
+ */
+ def testSparkSubmit(livyConf: LivyConf): Unit = {
+ try {
+ testSparkVersion(sparkSubmitVersion(livyConf)._1)
+ } catch {
+ case e: IOException =>
+ throw new IOException("Failed to run spark-submit executable", e)
+ }
+ }
+
+ /**
+ * Throw an exception if Spark version is not supported.
+ * @param version Spark version
+ */
+ def testSparkVersion(version: String): Unit = {
+ val v = formatSparkVersion(version)
+ require(v >= MIN_VERSION, s"Unsupported Spark version $v")
+ if (v >= MAX_VERSION) {
+ warn(s"Current Spark $v is not verified in Livy, please use it carefully")
+ }
+ }
+
+ /**
+ * Call `spark-submit --version` and parse its output for Spark and Scala version.
+ *
+ * @param livyConf
+ * @return Tuple with Spark and Scala version
+ */
+ def sparkSubmitVersion(livyConf: LivyConf): (String, Option[String]) = {
+ val sparkSubmit = livyConf.sparkSubmit()
+ val pb = new ProcessBuilder(sparkSubmit, "--version")
+ pb.redirectErrorStream(true)
+ pb.redirectInput(ProcessBuilder.Redirect.PIPE)
+
+ if (LivyConf.TEST_MODE) {
+ pb.environment().put("LIVY_TEST_CLASSPATH", sys.props("java.class.path"))
+ }
+
+ val process = new LineBufferedProcess(pb.start())
+ val exitCode = process.waitFor()
+ val output = process.inputIterator.mkString("\n")
+
+ var sparkVersion = ""
+ output match {
+ case sparkVersionRegex(version) => sparkVersion = version
+ case _ =>
+ throw new IOException(f"Unable to determine spark-submit version [$exitCode]:\n$output")
+ }
+
+ val scalaVersion = output match {
+ case scalaVersionRegex(version) if version.nonEmpty => Some(formatScalaVersion(version))
+ case _ => None
+ }
+
+ (sparkVersion, scalaVersion)
+ }
+
+ def sparkScalaVersion(
+ formattedSparkVersion: (Int, Int),
+ scalaVersionFromSparkSubmit: Option[String],
+ livyConf: LivyConf): String = {
+ val scalaVersionInLivyConf = Option(livyConf.get(LIVY_SPARK_SCALA_VERSION))
+ .filter(_.nonEmpty)
+ .map(formatScalaVersion)
+
+ for (vSparkSubmit <- scalaVersionFromSparkSubmit; vLivyConf <- scalaVersionInLivyConf) {
+ require(vSparkSubmit == vLivyConf,
+ s"Scala version detected from spark-submit ($vSparkSubmit) does not match " +
+ s"Scala version configured in livy.conf ($vLivyConf)")
+ }
+
+ scalaVersionInLivyConf
+ .orElse(scalaVersionFromSparkSubmit)
+ .getOrElse(defaultSparkScalaVersion(formattedSparkVersion))
+ }
+
+ /**
+ * Return formatted Spark version.
+ *
+ * @param version Spark version
+ * @return Two element tuple, one is major version and the other is minor version
+ */
+ def formatSparkVersion(version: String): (Int, Int) = {
+ val versionPattern = """^(\d+)\.(\d+)(\..*)?$""".r
+ versionPattern.findFirstMatchIn(version) match {
+ case Some(m) =>
+ (m.group(1).toInt, m.group(2).toInt)
+ case None =>
+ throw new IllegalArgumentException(s"Fail to parse Spark version from $version")
+ }
+ }
+
+ /**
+ * Return Scala binary version.
+ * It strips the patch version if specified.
+ * Throws if it cannot parse the version.
+ *
+ * @param scalaVersion Scala binary version String
+ * @return Scala binary version
+ */
+ def formatScalaVersion(scalaVersion: String): String = {
+ val versionPattern = """(\d)+\.(\d+)+.*""".r
+ scalaVersion match {
+ case versionPattern(major, minor) => s"$major.$minor"
+ case _ => throw new IllegalArgumentException(s"Unrecognized Scala version: $scalaVersion")
+ }
+ }
+
+ /**
+ * Return the default Scala version of a Spark version.
+ *
+ * @param sparkVersion formatted Spark version.
+ * @return Scala binary version
+ */
+ private[utils] def defaultSparkScalaVersion(sparkVersion: (Int, Int)): String = {
+ _defaultSparkScalaVersion.get(sparkVersion)
+ .orElse {
+ if (sparkVersion < _defaultSparkScalaVersion.head._1) {
+ throw new IllegalArgumentException(s"Spark version $sparkVersion is less than the " +
+ s"minimum version ${_defaultSparkScalaVersion.head._1} supported by Livy")
+ } else if (sparkVersion > _defaultSparkScalaVersion.last._1) {
+ val (spark, scala) = _defaultSparkScalaVersion.last
+ warn(s"Spark version $sparkVersion is greater then the maximum version " +
+ s"$spark supported by Livy, will choose Scala version $scala instead, " +
+ s"please specify manually if it is the expected Scala version you want")
+ Some(scala)
+ } else {
+ None
+ }
+ }
+ .getOrElse(
+ throw new IllegalArgumentException(s"Fail to get Scala version from Spark $sparkVersion"))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/utils/SparkApp.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/utils/SparkApp.scala b/server/src/main/scala/org/apache/livy/utils/SparkApp.scala
new file mode 100644
index 0000000..9afe281
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/utils/SparkApp.scala
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.utils
+
+import scala.collection.JavaConverters._
+
+import org.apache.livy.LivyConf
+
+object AppInfo {
+ val DRIVER_LOG_URL_NAME = "driverLogUrl"
+ val SPARK_UI_URL_NAME = "sparkUiUrl"
+}
+
+case class AppInfo(var driverLogUrl: Option[String] = None, var sparkUiUrl: Option[String] = None) {
+ import AppInfo._
+ def asJavaMap: java.util.Map[String, String] =
+ Map(DRIVER_LOG_URL_NAME -> driverLogUrl.orNull, SPARK_UI_URL_NAME -> sparkUiUrl.orNull).asJava
+}
+
+trait SparkAppListener {
+ /** Fired when appId is known, even during recovery. */
+ def appIdKnown(appId: String): Unit = {}
+
+ /** Fired when the app state in the cluster changes. */
+ def stateChanged(oldState: SparkApp.State, newState: SparkApp.State): Unit = {}
+
+ /** Fired when the app info is changed. */
+ def infoChanged(appInfo: AppInfo): Unit = {}
+}
+
+/**
+ * Provide factory methods for SparkApp.
+ */
+object SparkApp {
+ private val SPARK_YARN_TAG_KEY = "spark.yarn.tags"
+
+ object State extends Enumeration {
+ val STARTING, RUNNING, FINISHED, FAILED, KILLED = Value
+ }
+ type State = State.Value
+
+ /**
+ * Return cluster manager dependent SparkConf.
+ *
+ * @param uniqueAppTag A tag that can uniquely identify the application.
+ * @param livyConf
+ * @param sparkConf
+ */
+ def prepareSparkConf(
+ uniqueAppTag: String,
+ livyConf: LivyConf,
+ sparkConf: Map[String, String]): Map[String, String] = {
+ if (livyConf.isRunningOnYarn()) {
+ val userYarnTags = sparkConf.get(SPARK_YARN_TAG_KEY).map("," + _).getOrElse("")
+ val mergedYarnTags = uniqueAppTag + userYarnTags
+ sparkConf ++ Map(
+ SPARK_YARN_TAG_KEY -> mergedYarnTags,
+ "spark.yarn.submit.waitAppCompletion" -> "false")
+ } else {
+ sparkConf
+ }
+ }
+
+ /**
+ * Return a SparkApp object to control the underlying Spark application via YARN or spark-submit.
+ *
+ * @param uniqueAppTag A tag that can uniquely identify the application.
+ */
+ def create(
+ uniqueAppTag: String,
+ appId: Option[String],
+ process: Option[LineBufferedProcess],
+ livyConf: LivyConf,
+ listener: Option[SparkAppListener]): SparkApp = {
+ if (livyConf.isRunningOnYarn()) {
+ new SparkYarnApp(uniqueAppTag, appId, process, listener, livyConf)
+ } else {
+ require(process.isDefined, "process must not be None when Livy master is not YARN.")
+ new SparkProcApp(process.get, listener)
+ }
+ }
+}
+
+/**
+ * Encapsulate a Spark application.
+ */
+abstract class SparkApp {
+ def kill(): Unit
+ def log(): IndexedSeq[String]
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/utils/SparkProcApp.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/utils/SparkProcApp.scala b/server/src/main/scala/org/apache/livy/utils/SparkProcApp.scala
new file mode 100644
index 0000000..5fb3e42
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/utils/SparkProcApp.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.utils
+
+import org.apache.livy.{Logging, Utils}
+
+/**
+ * Provide a class to control a Spark application using spark-submit.
+ *
+ * @param process The spark-submit process launched the Spark application.
+ */
+class SparkProcApp (
+ process: LineBufferedProcess,
+ listener: Option[SparkAppListener])
+ extends SparkApp with Logging {
+
+ private var state = SparkApp.State.STARTING
+
+ override def kill(): Unit = {
+ if (process.isAlive) {
+ process.destroy()
+ waitThread.join()
+ }
+ }
+
+ override def log(): IndexedSeq[String] = process.inputLines
+
+ private def changeState(newState: SparkApp.State.Value) = {
+ if (state != newState) {
+ listener.foreach(_.stateChanged(state, newState))
+ state = newState
+ }
+ }
+
+ private val waitThread = Utils.startDaemonThread(s"SparProcApp_$this") {
+ changeState(SparkApp.State.RUNNING)
+ process.waitFor() match {
+ case 0 => changeState(SparkApp.State.FINISHED)
+ case exitCode =>
+ changeState(SparkApp.State.FAILED)
+ error(s"spark-submit exited with code $exitCode")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/utils/SparkProcessBuilder.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/utils/SparkProcessBuilder.scala b/server/src/main/scala/org/apache/livy/utils/SparkProcessBuilder.scala
new file mode 100644
index 0000000..66452d1
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/utils/SparkProcessBuilder.scala
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.utils
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.livy.{LivyConf, Logging}
+
+class SparkProcessBuilder(livyConf: LivyConf) extends Logging {
+
+ private[this] var _executable: String = livyConf.sparkSubmit()
+ private[this] var _master: Option[String] = None
+ private[this] var _deployMode: Option[String] = None
+ private[this] var _className: Option[String] = None
+ private[this] var _name: Option[String] = None
+ private[this] val _conf = mutable.HashMap[String, String]()
+ private[this] var _driverClassPath: ArrayBuffer[String] = ArrayBuffer()
+ private[this] var _proxyUser: Option[String] = None
+ private[this] var _queue: Option[String] = None
+ private[this] var _env: ArrayBuffer[(String, String)] = ArrayBuffer()
+ private[this] var _redirectOutput: Option[ProcessBuilder.Redirect] = None
+ private[this] var _redirectError: Option[ProcessBuilder.Redirect] = None
+ private[this] var _redirectErrorStream: Option[Boolean] = None
+
+ def executable(executable: String): SparkProcessBuilder = {
+ _executable = executable
+ this
+ }
+
+ def master(masterUrl: String): SparkProcessBuilder = {
+ _master = Some(masterUrl)
+ this
+ }
+
+ def deployMode(deployMode: String): SparkProcessBuilder = {
+ _deployMode = Some(deployMode)
+ this
+ }
+
+ def className(className: String): SparkProcessBuilder = {
+ _className = Some(className)
+ this
+ }
+
+ def name(name: String): SparkProcessBuilder = {
+ _name = Some(name)
+ this
+ }
+
+ def conf(key: String): Option[String] = {
+ _conf.get(key)
+ }
+
+ def conf(key: String, value: String, admin: Boolean = false): SparkProcessBuilder = {
+ this._conf(key) = value
+ this
+ }
+
+ def conf(conf: Traversable[(String, String)]): SparkProcessBuilder = {
+ conf.foreach { case (key, value) => this.conf(key, value) }
+ this
+ }
+
+ def driverJavaOptions(driverJavaOptions: String): SparkProcessBuilder = {
+ conf("spark.driver.extraJavaOptions", driverJavaOptions)
+ }
+
+ def driverClassPath(classPath: String): SparkProcessBuilder = {
+ _driverClassPath += classPath
+ this
+ }
+
+ def driverClassPaths(classPaths: Traversable[String]): SparkProcessBuilder = {
+ _driverClassPath ++= classPaths
+ this
+ }
+
+ def driverCores(driverCores: Int): SparkProcessBuilder = {
+ this.driverCores(driverCores.toString)
+ }
+
+ def driverMemory(driverMemory: String): SparkProcessBuilder = {
+ conf("spark.driver.memory", driverMemory)
+ }
+
+ def driverCores(driverCores: String): SparkProcessBuilder = {
+ conf("spark.driver.cores", driverCores)
+ }
+
+ def executorCores(executorCores: Int): SparkProcessBuilder = {
+ this.executorCores(executorCores.toString)
+ }
+
+ def executorCores(executorCores: String): SparkProcessBuilder = {
+ conf("spark.executor.cores", executorCores)
+ }
+
+ def executorMemory(executorMemory: String): SparkProcessBuilder = {
+ conf("spark.executor.memory", executorMemory)
+ }
+
+ def numExecutors(numExecutors: Int): SparkProcessBuilder = {
+ this.numExecutors(numExecutors.toString)
+ }
+
+ def numExecutors(numExecutors: String): SparkProcessBuilder = {
+ this.conf("spark.executor.instances", numExecutors)
+ }
+
+ def proxyUser(proxyUser: String): SparkProcessBuilder = {
+ _proxyUser = Some(proxyUser)
+ this
+ }
+
+ def queue(queue: String): SparkProcessBuilder = {
+ _queue = Some(queue)
+ this
+ }
+
+ def env(key: String, value: String): SparkProcessBuilder = {
+ _env += ((key, value))
+ this
+ }
+
+ def redirectOutput(redirect: ProcessBuilder.Redirect): SparkProcessBuilder = {
+ _redirectOutput = Some(redirect)
+ this
+ }
+
+ def redirectError(redirect: ProcessBuilder.Redirect): SparkProcessBuilder = {
+ _redirectError = Some(redirect)
+ this
+ }
+
+ def redirectErrorStream(redirect: Boolean): SparkProcessBuilder = {
+ _redirectErrorStream = Some(redirect)
+ this
+ }
+
+ def start(file: Option[String], args: Traversable[String]): LineBufferedProcess = {
+ var arguments = ArrayBuffer(_executable)
+
+ def addOpt(option: String, value: Option[String]): Unit = {
+ value.foreach { v =>
+ arguments += option
+ arguments += v
+ }
+ }
+
+ def addList(option: String, values: Traversable[String]): Unit = {
+ if (values.nonEmpty) {
+ arguments += option
+ arguments += values.mkString(",")
+ }
+ }
+
+ addOpt("--master", _master)
+ addOpt("--deploy-mode", _deployMode)
+ addOpt("--name", _name)
+ addOpt("--class", _className)
+ _conf.foreach { case (key, value) =>
+ if (key == "spark.submit.pyFiles") {
+ arguments += "--py-files"
+ arguments += f"$value"
+ } else {
+ arguments += "--conf"
+ arguments += f"$key=$value"
+ }
+ }
+ addList("--driver-class-path", _driverClassPath)
+
+ if (livyConf.getBoolean(LivyConf.IMPERSONATION_ENABLED)) {
+ addOpt("--proxy-user", _proxyUser)
+ }
+
+ addOpt("--queue", _queue)
+
+ arguments += file.getOrElse("spark-internal")
+ arguments ++= args
+
+ val argsString = arguments
+ .map("'" + _.replace("'", "\\'") + "'")
+ .mkString(" ")
+
+ info(s"Running $argsString")
+
+ val pb = new ProcessBuilder(arguments.asJava)
+ val env = pb.environment()
+
+ for ((key, value) <- _env) {
+ env.put(key, value)
+ }
+
+ _redirectOutput.foreach(pb.redirectOutput)
+ _redirectError.foreach(pb.redirectError)
+ _redirectErrorStream.foreach(pb.redirectErrorStream)
+
+ new LineBufferedProcess(pb.start())
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala b/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
new file mode 100644
index 0000000..b2a828f
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.utils
+
+import java.util.concurrent.TimeoutException
+
+import scala.annotation.tailrec
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+import scala.concurrent._
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.util.Try
+
+import org.apache.hadoop.yarn.api.records.{ApplicationId, ApplicationReport, FinalApplicationStatus, YarnApplicationState}
+import org.apache.hadoop.yarn.client.api.YarnClient
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException
+import org.apache.hadoop.yarn.util.ConverterUtils
+
+import org.apache.livy.{LivyConf, Logging, Utils}
+
+object SparkYarnApp extends Logging {
+
+ def init(livyConf: LivyConf): Unit = {
+ sessionLeakageCheckInterval = livyConf.getTimeAsMs(LivyConf.YARN_APP_LEAKAGE_CHECK_INTERVAL)
+ sessionLeakageCheckTimeout = livyConf.getTimeAsMs(LivyConf.YARN_APP_LEAKAGE_CHECK_TIMEOUT)
+ leakedAppsGCThread.setDaemon(true)
+ leakedAppsGCThread.setName("LeakedAppsGCThread")
+ leakedAppsGCThread.start()
+ }
+
+ // YarnClient is thread safe. Create once, share it across threads.
+ lazy val yarnClient = {
+ val c = YarnClient.createYarnClient()
+ c.init(new YarnConfiguration())
+ c.start()
+ c
+ }
+
+ private def getYarnTagToAppIdTimeout(livyConf: LivyConf): FiniteDuration =
+ livyConf.getTimeAsMs(LivyConf.YARN_APP_LOOKUP_TIMEOUT) milliseconds
+
+ private def getYarnPollInterval(livyConf: LivyConf): FiniteDuration =
+ livyConf.getTimeAsMs(LivyConf.YARN_POLL_INTERVAL) milliseconds
+
+ private val appType = Set("SPARK").asJava
+
+ private val leakedAppTags = new java.util.concurrent.ConcurrentHashMap[String, Long]()
+
+ private var sessionLeakageCheckTimeout: Long = _
+
+ private var sessionLeakageCheckInterval: Long = _
+
+ private val leakedAppsGCThread = new Thread() {
+ override def run(): Unit = {
+ while (true) {
+ if (!leakedAppTags.isEmpty) {
+ // kill the app if found it and remove it if exceeding a threashold
+ val iter = leakedAppTags.entrySet().iterator()
+ var isRemoved = false
+ val now = System.currentTimeMillis()
+ val apps = yarnClient.getApplications(appType).asScala
+ while(iter.hasNext) {
+ val entry = iter.next()
+ apps.find(_.getApplicationTags.contains(entry.getKey))
+ .foreach({ e =>
+ info(s"Kill leaked app ${e.getApplicationId}")
+ yarnClient.killApplication(e.getApplicationId)
+ iter.remove()
+ isRemoved = true
+ })
+ if (!isRemoved) {
+ if ((entry.getValue - now) > sessionLeakageCheckTimeout) {
+ iter.remove()
+ info(s"Remove leaked yarn app tag ${entry.getKey}")
+ }
+ }
+ }
+ }
+ Thread.sleep(sessionLeakageCheckInterval)
+ }
+ }
+ }
+
+
+}
+
+/**
+ * Provide a class to control a Spark application using YARN API.
+ *
+ * @param appTag An app tag that can unique identify the YARN app.
+ * @param appIdOption The appId of the YARN app. If this's None, SparkYarnApp will find it
+ * using appTag.
+ * @param process The spark-submit process launched the YARN application. This is optional.
+ * If it's provided, SparkYarnApp.log() will include its log.
+ * @param listener Optional listener for notification of appId discovery and app state changes.
+ */
+class SparkYarnApp private[utils] (
+ appTag: String,
+ appIdOption: Option[String],
+ process: Option[LineBufferedProcess],
+ listener: Option[SparkAppListener],
+ livyConf: LivyConf,
+ yarnClient: => YarnClient = SparkYarnApp.yarnClient) // For unit test.
+ extends SparkApp
+ with Logging {
+ import SparkYarnApp._
+
+ private val appIdPromise: Promise[ApplicationId] = Promise()
+ private[utils] var state: SparkApp.State = SparkApp.State.STARTING
+ private var yarnDiagnostics: IndexedSeq[String] = IndexedSeq.empty[String]
+
+ override def log(): IndexedSeq[String] =
+ ("stdout: " +: process.map(_.inputLines).getOrElse(ArrayBuffer.empty[String])) ++
+ ("\nstderr: " +: process.map(_.errorLines).getOrElse(ArrayBuffer.empty[String])) ++
+ ("\nYARN Diagnostics: " +: yarnDiagnostics)
+
+ override def kill(): Unit = synchronized {
+ if (isRunning) {
+ try {
+ val timeout = SparkYarnApp.getYarnTagToAppIdTimeout(livyConf)
+ yarnClient.killApplication(Await.result(appIdPromise.future, timeout))
+ } catch {
+ // We cannot kill the YARN app without the app id.
+ // There's a chance the YARN app hasn't been submitted during a livy-server failure.
+ // We don't want a stuck session that can't be deleted. Emit a warning and move on.
+ case _: TimeoutException | _: InterruptedException =>
+ warn("Deleting a session while its YARN application is not found.")
+ yarnAppMonitorThread.interrupt()
+ } finally {
+ process.foreach(_.destroy())
+ }
+ }
+ }
+
+ private def changeState(newState: SparkApp.State.Value): Unit = {
+ if (state != newState) {
+ listener.foreach(_.stateChanged(state, newState))
+ state = newState
+ }
+ }
+
+ /**
+ * Find the corresponding YARN application id from an application tag.
+ *
+ * @param appTag The application tag tagged on the target application.
+ * If the tag is not unique, it returns the first application it found.
+ * It will be converted to lower case to match YARN's behaviour.
+ * @return ApplicationId or the failure.
+ */
+ @tailrec
+ private def getAppIdFromTag(
+ appTag: String,
+ pollInterval: Duration,
+ deadline: Deadline): ApplicationId = {
+ val appTagLowerCase = appTag.toLowerCase()
+
+ // FIXME Should not loop thru all YARN applications but YarnClient doesn't offer an API.
+ // Consider calling rmClient in YarnClient directly.
+ yarnClient.getApplications(appType).asScala.find(_.getApplicationTags.contains(appTagLowerCase))
+ match {
+ case Some(app) => app.getApplicationId
+ case None =>
+ if (deadline.isOverdue) {
+ process.foreach(_.destroy())
+ leakedAppTags.put(appTag, System.currentTimeMillis())
+ throw new Exception(s"No YARN application is found with tag $appTagLowerCase in " +
+ livyConf.getTimeAsMs(LivyConf.YARN_APP_LOOKUP_TIMEOUT)/1000 + " seconds. " +
+ "Please check your cluster status, it is may be very busy.")
+ } else {
+ Clock.sleep(pollInterval.toMillis)
+ getAppIdFromTag(appTagLowerCase, pollInterval, deadline)
+ }
+ }
+ }
+
+ private def getYarnDiagnostics(appReport: ApplicationReport): IndexedSeq[String] = {
+ Option(appReport.getDiagnostics)
+ .filter(_.nonEmpty)
+ .map[IndexedSeq[String]]("YARN Diagnostics:" +: _.split("\n"))
+ .getOrElse(IndexedSeq.empty)
+ }
+
+ private def isRunning: Boolean = {
+ state != SparkApp.State.FAILED && state != SparkApp.State.FINISHED &&
+ state != SparkApp.State.KILLED
+ }
+
+ // Exposed for unit test.
+ private[utils] def mapYarnState(
+ appId: ApplicationId,
+ yarnAppState: YarnApplicationState,
+ finalAppStatus: FinalApplicationStatus): SparkApp.State.Value = {
+ yarnAppState match {
+ case (YarnApplicationState.NEW |
+ YarnApplicationState.NEW_SAVING |
+ YarnApplicationState.SUBMITTED |
+ YarnApplicationState.ACCEPTED) => SparkApp.State.STARTING
+ case YarnApplicationState.RUNNING => SparkApp.State.RUNNING
+ case YarnApplicationState.FINISHED =>
+ finalAppStatus match {
+ case FinalApplicationStatus.SUCCEEDED => SparkApp.State.FINISHED
+ case FinalApplicationStatus.FAILED => SparkApp.State.FAILED
+ case FinalApplicationStatus.KILLED => SparkApp.State.KILLED
+ case s =>
+ error(s"Unknown YARN final status $appId $s")
+ SparkApp.State.FAILED
+ }
+ case YarnApplicationState.FAILED => SparkApp.State.FAILED
+ case YarnApplicationState.KILLED => SparkApp.State.KILLED
+ }
+ }
+
+ // Exposed for unit test.
+ // TODO Instead of spawning a thread for every session, create a centralized thread and
+ // batch YARN queries.
+ private[utils] val yarnAppMonitorThread = Utils.startDaemonThread(s"yarnAppMonitorThread-$this") {
+ try {
+ // Wait for spark-submit to finish submitting the app to YARN.
+ process.foreach { p =>
+ val exitCode = p.waitFor()
+ if (exitCode != 0) {
+ throw new Exception(s"spark-submit exited with code $exitCode}.\n" +
+ s"${process.get.inputLines.mkString("\n")}")
+ }
+ }
+
+ // If appId is not known, query YARN by appTag to get it.
+ val appId = try {
+ appIdOption.map(ConverterUtils.toApplicationId).getOrElse {
+ val pollInterval = getYarnPollInterval(livyConf)
+ val deadline = getYarnTagToAppIdTimeout(livyConf).fromNow
+ getAppIdFromTag(appTag, pollInterval, deadline)
+ }
+ } catch {
+ case e: Exception =>
+ appIdPromise.failure(e)
+ throw e
+ }
+ appIdPromise.success(appId)
+
+ Thread.currentThread().setName(s"yarnAppMonitorThread-$appId")
+ listener.foreach(_.appIdKnown(appId.toString))
+
+ val pollInterval = SparkYarnApp.getYarnPollInterval(livyConf)
+ var appInfo = AppInfo()
+ while (isRunning) {
+ try {
+ Clock.sleep(pollInterval.toMillis)
+
+ // Refresh application state
+ val appReport = yarnClient.getApplicationReport(appId)
+ yarnDiagnostics = getYarnDiagnostics(appReport)
+ changeState(mapYarnState(
+ appReport.getApplicationId,
+ appReport.getYarnApplicationState,
+ appReport.getFinalApplicationStatus))
+
+ val latestAppInfo = {
+ val attempt =
+ yarnClient.getApplicationAttemptReport(appReport.getCurrentApplicationAttemptId)
+ val driverLogUrl =
+ Try(yarnClient.getContainerReport(attempt.getAMContainerId).getLogUrl)
+ .toOption
+ AppInfo(driverLogUrl, Option(appReport.getTrackingUrl))
+ }
+
+ if (appInfo != latestAppInfo) {
+ listener.foreach(_.infoChanged(latestAppInfo))
+ appInfo = latestAppInfo
+ }
+ } catch {
+ // This exception might be thrown during app is starting up. It's transient.
+ case e: ApplicationAttemptNotFoundException =>
+ // Workaround YARN-4411: No enum constant FINAL_SAVING from getApplicationAttemptReport()
+ case e: IllegalArgumentException =>
+ if (e.getMessage.contains("FINAL_SAVING")) {
+ debug("Encountered YARN-4411.")
+ } else {
+ throw e
+ }
+ }
+ }
+
+ debug(s"$appId $state ${yarnDiagnostics.mkString(" ")}")
+ } catch {
+ case e: InterruptedException =>
+ yarnDiagnostics = ArrayBuffer("Session stopped by user.")
+ changeState(SparkApp.State.KILLED)
+ case e: Throwable =>
+ error(s"Error whiling refreshing YARN state: $e")
+ yarnDiagnostics = ArrayBuffer(e.toString, e.getStackTrace().mkString(" "))
+ changeState(SparkApp.State.FAILED)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/ApiVersioningSupportSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/ApiVersioningSupportSpec.scala b/server/src/test/scala/com/cloudera/livy/server/ApiVersioningSupportSpec.scala
deleted file mode 100644
index 4c530f3..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/ApiVersioningSupportSpec.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-import javax.servlet.http.HttpServletResponse
-
-import org.scalatest.FunSpecLike
-import org.scalatra.ScalatraServlet
-import org.scalatra.test.scalatest.ScalatraSuite
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-
-class ApiVersioningSupportSpec extends ScalatraSuite with FunSpecLike with LivyBaseUnitTestSuite {
- val LatestVersionOutput = "latest"
-
- object FakeApiVersions extends Enumeration {
- type FakeApiVersions = Value
- val v0_1 = Value("0.1")
- val v0_2 = Value("0.2")
- val v1_0 = Value("1.0")
- }
-
- import FakeApiVersions._
-
- class MockServlet extends ScalatraServlet with AbstractApiVersioningSupport {
- override val apiVersions = FakeApiVersions
- override type ApiVersionType = FakeApiVersions.Value
-
- get("/test") {
- response.writer.write(LatestVersionOutput)
- }
-
- get("/test", apiVersion <= v0_2) {
- response.writer.write(v0_2.toString)
- }
-
- get("/test", apiVersion <= v0_1) {
- response.writer.write(v0_1.toString)
- }
-
- get("/droppedApi", apiVersion <= v0_2) {
- }
-
- get("/newApi", apiVersion >= v0_2) {
- }
- }
-
- var mockServlet: MockServlet = new MockServlet
- addServlet(mockServlet, "/*")
-
- def generateHeader(acceptHeader: String): Map[String, String] = {
- if (acceptHeader != null) Map("Accept" -> acceptHeader) else Map.empty
- }
-
- def shouldReturn(url: String, acceptHeader: String, expectedVersion: String = null): Unit = {
- get(url, headers = generateHeader(acceptHeader)) {
- status should equal(200)
- if (expectedVersion != null) {
- body should equal(expectedVersion)
- }
- }
- }
-
- def shouldFail(url: String, acceptHeader: String, expectedErrorCode: Int): Unit = {
- get(url, headers = generateHeader(acceptHeader)) {
- status should equal(expectedErrorCode)
- }
- }
-
- it("should pick the latest API version if Accept header is unspecified") {
- shouldReturn("/test", null, LatestVersionOutput)
- }
-
- it("should pick the latest API version if Accept header does not specify any version") {
- shouldReturn("/test", "foo", LatestVersionOutput)
- shouldReturn("/test", "application/vnd.random.v1.1", LatestVersionOutput)
- shouldReturn("/test", "application/vnd.livy.+json", LatestVersionOutput)
- }
-
- it("should pick the correct API version") {
- shouldReturn("/test", "application/vnd.livy.v0.1", v0_1.toString)
- shouldReturn("/test", "application/vnd.livy.v0.2+", v0_2.toString)
- shouldReturn("/test", "application/vnd.livy.v0.1+bar", v0_1.toString)
- shouldReturn("/test", "application/vnd.livy.v0.2+foo", v0_2.toString)
- shouldReturn("/test", "application/vnd.livy.v0.1+vnd.livy.v0.2", v0_1.toString)
- shouldReturn("/test", "application/vnd.livy.v0.2++++++++++++++++", v0_2.toString)
- shouldReturn("/test", "application/vnd.livy.v1.0", LatestVersionOutput)
- }
-
- it("should return error when the specified API version does not exist") {
- shouldFail("/test", "application/vnd.livy.v", HttpServletResponse.SC_NOT_ACCEPTABLE)
- shouldFail("/test", "application/vnd.livy.v+json", HttpServletResponse.SC_NOT_ACCEPTABLE)
- shouldFail("/test", "application/vnd.livy.v666.666", HttpServletResponse.SC_NOT_ACCEPTABLE)
- shouldFail("/test", "application/vnd.livy.v666.666+json", HttpServletResponse.SC_NOT_ACCEPTABLE)
- shouldFail("/test", "application/vnd.livy.v1.1+json", HttpServletResponse.SC_NOT_ACCEPTABLE)
- }
-
- it("should not see a dropped API") {
- shouldReturn("/droppedApi", "application/vnd.livy.v0.1+json")
- shouldReturn("/droppedApi", "application/vnd.livy.v0.2+json")
- shouldFail("/droppedApi", "application/vnd.livy.v1.0+json", HttpServletResponse.SC_NOT_FOUND)
- }
-
- it("should not see a new API at an older version") {
- shouldFail("/newApi", "application/vnd.livy.v0.1+json", HttpServletResponse.SC_NOT_FOUND)
- shouldReturn("/newApi", "application/vnd.livy.v0.2+json")
- shouldReturn("/newApi", "application/vnd.livy.v1.0+json")
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/BaseJsonServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/BaseJsonServletSpec.scala b/server/src/test/scala/com/cloudera/livy/server/BaseJsonServletSpec.scala
deleted file mode 100644
index 1d8d38a..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/BaseJsonServletSpec.scala
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-import java.io.ByteArrayOutputStream
-import javax.servlet.http.HttpServletResponse._
-
-import scala.reflect.ClassTag
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import org.scalatest.FunSpecLike
-import org.scalatra.test.scalatest.ScalatraSuite
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-
-/**
- * Base class that enhances ScalatraSuite so that it's easier to test JsonServlet
- * implementations. Variants of the test methods (get, post, etc) exist with the "j"
- * prefix; these automatically serialize the body of the request to JSON, and
- * deserialize the result from JSON.
- *
- * In case the response is not JSON, the expected type for the test function should be
- * `Unit`, and the `response` object should be checked directly.
- */
-abstract class BaseJsonServletSpec extends ScalatraSuite
- with FunSpecLike with LivyBaseUnitTestSuite {
-
- protected val mapper = new ObjectMapper()
- .registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
-
- protected val defaultHeaders: Map[String, String] = Map("Content-Type" -> "application/json")
-
- protected def jdelete[R: ClassTag](
- uri: String,
- expectedStatus: Int = SC_OK,
- headers: Map[String, String] = defaultHeaders)
- (fn: R => Unit): Unit = {
- delete(uri, headers = headers)(doTest(expectedStatus, fn))
- }
-
- protected def jget[R: ClassTag](
- uri: String,
- expectedStatus: Int = SC_OK,
- headers: Map[String, String] = defaultHeaders)
- (fn: R => Unit): Unit = {
- get(uri, headers = headers)(doTest(expectedStatus, fn))
- }
-
- protected def jpatch[R: ClassTag](
- uri: String,
- body: AnyRef,
- expectedStatus: Int = SC_OK,
- headers: Map[String, String] = defaultHeaders)
- (fn: R => Unit): Unit = {
- patch(uri, body = toJson(body), headers = headers)(doTest(expectedStatus, fn))
- }
-
- protected def jpost[R: ClassTag](
- uri: String,
- body: AnyRef,
- expectedStatus: Int = SC_CREATED,
- headers: Map[String, String] = defaultHeaders)
- (fn: R => Unit): Unit = {
- post(uri, body = toJson(body), headers = headers)(doTest(expectedStatus, fn))
- }
-
- /** A version of jpost specific for testing file upload. */
- protected def jupload[R: ClassTag](
- uri: String,
- files: Iterable[(String, Any)],
- headers: Map[String, String] = Map(),
- expectedStatus: Int = SC_OK)
- (fn: R => Unit): Unit = {
- post(uri, Map.empty, files)(doTest(expectedStatus, fn))
- }
-
- protected def jput[R: ClassTag](
- uri: String,
- body: AnyRef,
- expectedStatus: Int = SC_OK,
- headers: Map[String, String] = defaultHeaders)
- (fn: R => Unit): Unit = {
- put(uri, body = toJson(body), headers = headers)(doTest(expectedStatus, fn))
- }
-
- private def doTest[R: ClassTag](expectedStatus: Int, fn: R => Unit)
- (implicit klass: ClassTag[R]): Unit = {
- if (status != expectedStatus) {
- // Yeah this is weird, but we don't want to evaluate "response.body" if there's no error.
- assert(status === expectedStatus,
- s"Unexpected response status: $status != $expectedStatus (${response.body})")
- }
- // Only try to parse the body if response is in the "OK" range (20x).
- if ((status / 100) * 100 == SC_OK) {
- val result =
- if (header("Content-Type").startsWith("application/json")) {
- // Sometimes there's an empty body with no "Content-Length" header. So read the whole
- // body first, and only send it to Jackson if there's content.
- val in = response.inputStream
- val out = new ByteArrayOutputStream()
- val buf = new Array[Byte](1024)
- var read = 0
- while (read >= 0) {
- read = in.read(buf)
- if (read > 0) {
- out.write(buf, 0, read)
- }
- }
-
- val data = out.toByteArray()
- if (data.length > 0) {
- mapper.readValue(data, klass.runtimeClass)
- } else {
- null
- }
- } else {
- assert(klass.runtimeClass == classOf[Unit])
- ()
- }
- fn(result.asInstanceOf[R])
- }
- }
-
- private def toJson(obj: Any): Array[Byte] = mapper.writeValueAsBytes(obj)
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/BaseSessionServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/BaseSessionServletSpec.scala b/server/src/test/scala/com/cloudera/livy/server/BaseSessionServletSpec.scala
deleted file mode 100644
index 14eb2e6..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/BaseSessionServletSpec.scala
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-import javax.servlet.http.HttpServletRequest
-
-import org.scalatest.BeforeAndAfterAll
-
-import com.cloudera.livy.LivyConf
-import com.cloudera.livy.sessions.Session
-import com.cloudera.livy.sessions.Session.RecoveryMetadata
-
-object BaseSessionServletSpec {
-
- /** Header used to override the user remote user in tests. */
- val REMOTE_USER_HEADER = "X-Livy-SessionServlet-User"
-
-}
-
-abstract class BaseSessionServletSpec[S <: Session, R <: RecoveryMetadata]
- extends BaseJsonServletSpec
- with BeforeAndAfterAll {
-
- /** Config map containing option that is blacklisted. */
- protected val BLACKLISTED_CONFIG = Map("spark.do_not_set" -> "true")
-
- /** Name of the admin user. */
- protected val ADMIN = "__admin__"
-
- /** Create headers that identify a specific user in tests. */
- protected def makeUserHeaders(user: String): Map[String, String] = {
- defaultHeaders ++ Map(BaseSessionServletSpec.REMOTE_USER_HEADER -> user)
- }
-
- protected val adminHeaders = makeUserHeaders(ADMIN)
-
- /** Create a LivyConf with impersonation enabled and a superuser. */
- protected def createConf(): LivyConf = {
- new LivyConf()
- .set(LivyConf.IMPERSONATION_ENABLED, true)
- .set(LivyConf.SUPERUSERS, ADMIN)
- .set(LivyConf.LOCAL_FS_WHITELIST, sys.props("java.io.tmpdir"))
- }
-
- override def afterAll(): Unit = {
- super.afterAll()
- servlet.shutdown()
- }
-
- def createServlet(): SessionServlet[S, R]
-
- protected val servlet = createServlet()
-
- addServlet(servlet, "/*")
-
- protected def toJson(msg: AnyRef): Array[Byte] = mapper.writeValueAsBytes(msg)
-
-}
-
-trait RemoteUserOverride {
- this: SessionServlet[_, _] =>
-
- override protected def remoteUser(req: HttpServletRequest): String = {
- req.getHeader(BaseSessionServletSpec.REMOTE_USER_HEADER)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/JsonServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/JsonServletSpec.scala b/server/src/test/scala/com/cloudera/livy/server/JsonServletSpec.scala
deleted file mode 100644
index 3713443..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/JsonServletSpec.scala
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-import java.nio.charset.StandardCharsets.UTF_8
-import javax.servlet.http.HttpServletResponse._
-
-import org.scalatra._
-
-class JsonServletSpec extends BaseJsonServletSpec {
-
- addServlet(new TestJsonServlet(), "/*")
-
- describe("The JSON servlet") {
-
- it("should serialize result of delete") {
- jdelete[MethodReturn]("/delete") { result =>
- assert(result.value === "delete")
- }
- }
-
- it("should serialize result of get") {
- jget[MethodReturn]("/get") { result =>
- assert(result.value === "get")
- }
- }
-
- it("should serialize an ActionResult's body") {
- jpost[MethodReturn]("/post", MethodArg("post")) { result =>
- assert(result.value === "post")
- }
- }
-
- it("should wrap a raw result") {
- jput[MethodReturn]("/put", MethodArg("put")) { result =>
- assert(result.value === "put")
- }
- }
-
- it("should bypass non-json results") {
- jpatch[Unit]("/patch", MethodArg("patch"), expectedStatus = SC_NOT_FOUND) { _ =>
- assert(response.body === "patch")
- }
- }
-
- it("should translate JSON errors to BadRequest") {
- post("/post", "abcde".getBytes(UTF_8), headers = defaultHeaders) {
- assert(status === SC_BAD_REQUEST)
- }
- }
-
- it("should translate bad param name to BadRequest") {
- post("/post", """{"value1":"1"}""".getBytes(UTF_8), headers = defaultHeaders) {
- assert(status === SC_BAD_REQUEST)
- }
- }
-
- it("should translate type mismatch to BadRequest") {
- post("/postlist", """{"listParam":"1"}""".getBytes(UTF_8), headers = defaultHeaders) {
- assert(status === SC_BAD_REQUEST)
- }
- }
-
- it("should respect user-installed error handlers") {
- post("/error", headers = defaultHeaders) {
- assert(status === SC_SERVICE_UNAVAILABLE)
- assert(response.body === "error")
- }
- }
-
- it("should handle empty return values") {
- jget[MethodReturn]("/empty") { result =>
- assert(result == null)
- }
- }
-
- }
-
-}
-
-private case class MethodArg(value: String)
-
-private case class MethodReturn(value: String)
-
-private case class MethodReturnList(listParam: List[String] = List())
-
-private class TestJsonServlet extends JsonServlet {
-
- before() {
- contentType = "application/json"
- }
-
- delete("/delete") {
- Ok(MethodReturn("delete"))
- }
-
- get("/get") {
- Ok(MethodReturn("get"))
- }
-
- jpost[MethodArg]("/post") { arg =>
- Created(MethodReturn(arg.value))
- }
-
- jpost[MethodReturnList]("/postlist") { arg =>
- Created()
- }
-
- jput[MethodArg]("/put") { arg =>
- MethodReturn(arg.value)
- }
-
- jpatch[MethodArg]("/patch") { arg =>
- contentType = "text/plain"
- NotFound(arg.value)
- }
-
- get("/empty") {
- ()
- }
-
- post("/error") {
- throw new IllegalStateException("error")
- }
-
- // Install an error handler to make sure the parent's still work.
- error {
- case e: IllegalStateException =>
- contentType = "text/plain"
- ServiceUnavailable(e.getMessage())
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/SessionServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/SessionServletSpec.scala b/server/src/test/scala/com/cloudera/livy/server/SessionServletSpec.scala
deleted file mode 100644
index 1ae8a25..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/SessionServletSpec.scala
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server
-
-
-import javax.servlet.http.HttpServletRequest
-import javax.servlet.http.HttpServletResponse._
-
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.LivyConf
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.{Session, SessionManager, SessionState}
-import com.cloudera.livy.sessions.Session.RecoveryMetadata
-
-object SessionServletSpec {
-
- val PROXY_USER = "proxyUser"
-
- class MockSession(id: Int, owner: String, livyConf: LivyConf)
- extends Session(id, owner, livyConf) {
-
- case class MockRecoveryMetadata(id: Int) extends RecoveryMetadata()
-
- override val proxyUser = None
-
- override def recoveryMetadata: RecoveryMetadata = MockRecoveryMetadata(0)
-
- override def state: SessionState = SessionState.Idle()
-
- override protected def stopSession(): Unit = ()
-
- override def logLines(): IndexedSeq[String] = IndexedSeq("log")
-
- }
-
- case class MockSessionView(id: Int, owner: String, logs: Seq[String])
-
-}
-
-class SessionServletSpec
- extends BaseSessionServletSpec[Session, RecoveryMetadata] {
-
- import SessionServletSpec._
-
- override def createServlet(): SessionServlet[Session, RecoveryMetadata] = {
- val conf = createConf()
- val sessionManager = new SessionManager[Session, RecoveryMetadata](
- conf,
- { _ => assert(false).asInstanceOf[Session] },
- mock[SessionStore],
- "test",
- Some(Seq.empty))
-
- new SessionServlet(sessionManager, conf) with RemoteUserOverride {
- override protected def createSession(req: HttpServletRequest): Session = {
- val params = bodyAs[Map[String, String]](req)
- checkImpersonation(params.get(PROXY_USER), req)
- new MockSession(sessionManager.nextId(), remoteUser(req), conf)
- }
-
- override protected def clientSessionView(
- session: Session,
- req: HttpServletRequest): Any = {
- val logs = if (hasAccess(session.owner, req)) session.logLines() else Nil
- MockSessionView(session.id, session.owner, logs)
- }
- }
- }
-
- private val aliceHeaders = makeUserHeaders("alice")
- private val bobHeaders = makeUserHeaders("bob")
-
- private def delete(id: Int, headers: Map[String, String], expectedStatus: Int): Unit = {
- jdelete[Map[String, Any]](s"/$id", headers = headers, expectedStatus = expectedStatus) { _ =>
- // Nothing to do.
- }
- }
-
- describe("SessionServlet") {
-
- it("should return correct Location in header") {
- // mount to "/sessions/*" to test. If request URI is "/session", getPathInfo() will
- // return null, since there's no extra path.
- // mount to "/*" will always return "/", so that it cannot reflect the issue.
- addServlet(servlet, "/sessions/*")
- jpost[MockSessionView]("/sessions", Map(), headers = aliceHeaders) { res =>
- assert(header("Location") === "/sessions/0")
- jdelete[Map[String, Any]]("/sessions/0", SC_OK, aliceHeaders) { _ => }
- }
- }
-
- it("should attach owner information to sessions") {
- jpost[MockSessionView]("/", Map(), headers = aliceHeaders) { res =>
- assert(res.owner === "alice")
- assert(res.logs === IndexedSeq("log"))
- delete(res.id, aliceHeaders, SC_OK)
- }
- }
-
- it("should allow other users to see non-sensitive information") {
- jpost[MockSessionView]("/", Map(), headers = aliceHeaders) { res =>
- jget[MockSessionView](s"/${res.id}", headers = bobHeaders) { res =>
- assert(res.owner === "alice")
- assert(res.logs === Nil)
- }
- delete(res.id, aliceHeaders, SC_OK)
- }
- }
-
- it("should prevent non-owners from modifying sessions") {
- jpost[MockSessionView]("/", Map(), headers = aliceHeaders) { res =>
- delete(res.id, bobHeaders, SC_FORBIDDEN)
- }
- }
-
- it("should allow admins to access all sessions") {
- jpost[MockSessionView]("/", Map(), headers = aliceHeaders) { res =>
- jget[MockSessionView](s"/${res.id}", headers = adminHeaders) { res =>
- assert(res.owner === "alice")
- assert(res.logs === IndexedSeq("log"))
- }
- delete(res.id, adminHeaders, SC_OK)
- }
- }
-
- it("should not allow regular users to impersonate others") {
- jpost[MockSessionView]("/", Map(PROXY_USER -> "bob"), headers = aliceHeaders,
- expectedStatus = SC_FORBIDDEN) { _ => }
- }
-
- it("should allow admins to impersonate anyone") {
- jpost[MockSessionView]("/", Map(PROXY_USER -> "bob"), headers = adminHeaders) { res =>
- delete(res.id, bobHeaders, SC_FORBIDDEN)
- delete(res.id, adminHeaders, SC_OK)
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/batch/BatchServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/batch/BatchServletSpec.scala b/server/src/test/scala/com/cloudera/livy/server/batch/BatchServletSpec.scala
deleted file mode 100644
index 8a79593..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/batch/BatchServletSpec.scala
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.batch
-
-import java.io.FileWriter
-import java.nio.file.{Files, Path}
-import java.util.concurrent.TimeUnit
-import javax.servlet.http.HttpServletRequest
-import javax.servlet.http.HttpServletResponse._
-
-import scala.concurrent.duration.Duration
-
-import org.mockito.Mockito._
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.Utils
-import com.cloudera.livy.server.BaseSessionServletSpec
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.{BatchSessionManager, SessionState}
-import com.cloudera.livy.utils.AppInfo
-
-class BatchServletSpec extends BaseSessionServletSpec[BatchSession, BatchRecoveryMetadata] {
-
- val script: Path = {
- val script = Files.createTempFile("livy-test", ".py")
- script.toFile.deleteOnExit()
- val writer = new FileWriter(script.toFile)
- try {
- writer.write(
- """
- |print "hello world"
- """.stripMargin)
- } finally {
- writer.close()
- }
- script
- }
-
- override def createServlet(): BatchSessionServlet = {
- val livyConf = createConf()
- val sessionStore = mock[SessionStore]
- new BatchSessionServlet(
- new BatchSessionManager(livyConf, sessionStore, Some(Seq.empty)),
- sessionStore,
- livyConf)
- }
-
- describe("Batch Servlet") {
- it("should create and tear down a batch") {
- jget[Map[String, Any]]("/") { data =>
- data("sessions") should equal (Seq())
- }
-
- val createRequest = new CreateBatchRequest()
- createRequest.file = script.toString
- createRequest.conf = Map("spark.driver.extraClassPath" -> sys.props("java.class.path"))
-
- jpost[Map[String, Any]]("/", createRequest) { data =>
- header("Location") should equal("/0")
- data("id") should equal (0)
-
- val batch = servlet.sessionManager.get(0)
- batch should be (defined)
- }
-
- // Wait for the process to finish.
- {
- val batch = servlet.sessionManager.get(0).get
- Utils.waitUntil({ () => !batch.state.isActive }, Duration(10, TimeUnit.SECONDS))
- (batch.state match {
- case SessionState.Success(_) => true
- case _ => false
- }) should be (true)
- }
-
- jget[Map[String, Any]]("/0") { data =>
- data("id") should equal (0)
- data("state") should equal ("success")
-
- val batch = servlet.sessionManager.get(0)
- batch should be (defined)
- }
-
- jget[Map[String, Any]]("/0/log?size=1000") { data =>
- data("id") should equal (0)
- data("log").asInstanceOf[Seq[String]] should contain ("hello world")
-
- val batch = servlet.sessionManager.get(0)
- batch should be (defined)
- }
-
- jdelete[Map[String, Any]]("/0") { data =>
- data should equal (Map("msg" -> "deleted"))
-
- val batch = servlet.sessionManager.get(0)
- batch should not be defined
- }
- }
-
- it("should respect config black list") {
- val createRequest = new CreateBatchRequest()
- createRequest.file = script.toString
- createRequest.conf = BLACKLISTED_CONFIG
- jpost[Map[String, Any]]("/", createRequest, expectedStatus = SC_BAD_REQUEST) { _ => }
- }
-
- it("should show session properties") {
- val id = 0
- val state = SessionState.Running()
- val appId = "appid"
- val appInfo = AppInfo(Some("DRIVER LOG URL"), Some("SPARK UI URL"))
- val log = IndexedSeq[String]("log1", "log2")
-
- val session = mock[BatchSession]
- when(session.id).thenReturn(id)
- when(session.state).thenReturn(state)
- when(session.appId).thenReturn(Some(appId))
- when(session.appInfo).thenReturn(appInfo)
- when(session.logLines()).thenReturn(log)
-
- val req = mock[HttpServletRequest]
-
- val view = servlet.asInstanceOf[BatchSessionServlet].clientSessionView(session, req)
- .asInstanceOf[BatchSessionView]
-
- view.id shouldEqual id
- view.state shouldEqual state.toString
- view.appId shouldEqual Some(appId)
- view.appInfo shouldEqual appInfo
- view.log shouldEqual log
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/batch/BatchSessionSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/batch/BatchSessionSpec.scala b/server/src/test/scala/com/cloudera/livy/server/batch/BatchSessionSpec.scala
deleted file mode 100644
index 0aa8d28..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/batch/BatchSessionSpec.scala
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.batch
-
-import java.io.FileWriter
-import java.nio.file.{Files, Path}
-import java.util.concurrent.TimeUnit
-
-import scala.concurrent.duration.Duration
-
-import org.mockito.Matchers
-import org.mockito.Matchers.anyObject
-import org.mockito.Mockito._
-import org.scalatest.{BeforeAndAfter, FunSpec, ShouldMatchers}
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf, Utils}
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.SessionState
-import com.cloudera.livy.utils.{AppInfo, SparkApp}
-
-class BatchSessionSpec
- extends FunSpec
- with BeforeAndAfter
- with ShouldMatchers
- with LivyBaseUnitTestSuite {
-
- val script: Path = {
- val script = Files.createTempFile("livy-test", ".py")
- script.toFile.deleteOnExit()
- val writer = new FileWriter(script.toFile)
- try {
- writer.write(
- """
- |print "hello world"
- """.stripMargin)
- } finally {
- writer.close()
- }
- script
- }
-
- describe("A Batch process") {
- var sessionStore: SessionStore = null
-
- before {
- sessionStore = mock[SessionStore]
- }
-
- it("should create a process") {
- val req = new CreateBatchRequest()
- req.file = script.toString
- req.conf = Map("spark.driver.extraClassPath" -> sys.props("java.class.path"))
-
- val conf = new LivyConf().set(LivyConf.LOCAL_FS_WHITELIST, sys.props("java.io.tmpdir"))
- val batch = BatchSession.create(0, req, conf, null, None, sessionStore)
-
- Utils.waitUntil({ () => !batch.state.isActive }, Duration(10, TimeUnit.SECONDS))
- (batch.state match {
- case SessionState.Success(_) => true
- case _ => false
- }) should be (true)
-
- batch.logLines() should contain("hello world")
- }
-
- it("should update appId and appInfo") {
- val conf = new LivyConf()
- val req = new CreateBatchRequest()
- val mockApp = mock[SparkApp]
- val batch = BatchSession.create(0, req, conf, null, None, sessionStore, Some(mockApp))
-
- val expectedAppId = "APPID"
- batch.appIdKnown(expectedAppId)
- verify(sessionStore, atLeastOnce()).save(
- Matchers.eq(BatchSession.RECOVERY_SESSION_TYPE), anyObject())
- batch.appId shouldEqual Some(expectedAppId)
-
- val expectedAppInfo = AppInfo(Some("DRIVER LOG URL"), Some("SPARK UI URL"))
- batch.infoChanged(expectedAppInfo)
- batch.appInfo shouldEqual expectedAppInfo
- }
-
- it("should recover session") {
- val conf = new LivyConf()
- val req = new CreateBatchRequest()
- val mockApp = mock[SparkApp]
- val m = BatchRecoveryMetadata(99, None, "appTag", null, None)
- val batch = BatchSession.recover(m, conf, sessionStore, Some(mockApp))
-
- batch.state shouldBe a[SessionState.Recovering]
-
- batch.appIdKnown("appId")
- verify(sessionStore, atLeastOnce()).save(
- Matchers.eq(BatchSession.RECOVERY_SESSION_TYPE), anyObject())
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/batch/CreateBatchRequestSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/batch/CreateBatchRequestSpec.scala b/server/src/test/scala/com/cloudera/livy/server/batch/CreateBatchRequestSpec.scala
deleted file mode 100644
index 9119e79..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/batch/CreateBatchRequestSpec.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.batch
-
-import com.fasterxml.jackson.databind.{JsonMappingException, ObjectMapper}
-import org.scalatest.FunSpec
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-
-class CreateBatchRequestSpec extends FunSpec with LivyBaseUnitTestSuite {
-
- private val mapper = new ObjectMapper()
- .registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
-
- describe("CreateBatchRequest") {
-
- it("should have default values for fields after deserialization") {
- val json = """{ "file" : "foo" }"""
- val req = mapper.readValue(json, classOf[CreateBatchRequest])
- assert(req.file === "foo")
- assert(req.proxyUser === None)
- assert(req.args === List())
- assert(req.className === None)
- assert(req.jars === List())
- assert(req.pyFiles === List())
- assert(req.files === List())
- assert(req.driverMemory === None)
- assert(req.driverCores === None)
- assert(req.executorMemory === None)
- assert(req.executorCores === None)
- assert(req.numExecutors === None)
- assert(req.archives === List())
- assert(req.queue === None)
- assert(req.name === None)
- assert(req.conf === Map())
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/interactive/BaseInteractiveServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/interactive/BaseInteractiveServletSpec.scala b/server/src/test/scala/com/cloudera/livy/server/interactive/BaseInteractiveServletSpec.scala
deleted file mode 100644
index fc48643..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/interactive/BaseInteractiveServletSpec.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.interactive
-
-import java.io.File
-import java.nio.file.Files
-
-import org.apache.commons.io.FileUtils
-import org.apache.spark.launcher.SparkLauncher
-
-import com.cloudera.livy.LivyConf
-import com.cloudera.livy.rsc.RSCConf
-import com.cloudera.livy.server.BaseSessionServletSpec
-import com.cloudera.livy.sessions.{Kind, SessionKindModule, Spark}
-
-abstract class BaseInteractiveServletSpec
- extends BaseSessionServletSpec[InteractiveSession, InteractiveRecoveryMetadata] {
-
- mapper.registerModule(new SessionKindModule())
-
- protected var tempDir: File = _
-
- override def afterAll(): Unit = {
- super.afterAll()
- if (tempDir != null) {
- scala.util.Try(FileUtils.deleteDirectory(tempDir))
- tempDir = null
- }
- }
-
- override protected def createConf(): LivyConf = synchronized {
- if (tempDir == null) {
- tempDir = Files.createTempDirectory("client-test").toFile()
- }
- super.createConf()
- .set(LivyConf.SESSION_STAGING_DIR, tempDir.toURI().toString())
- .set(LivyConf.REPL_JARS, "dummy.jar")
- .set(LivyConf.LIVY_SPARK_VERSION, "1.6.0")
- .set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.10.5")
- }
-
- protected def createRequest(
- inProcess: Boolean = true,
- extraConf: Map[String, String] = Map(),
- kind: Kind = Spark()): CreateInteractiveRequest = {
- val classpath = sys.props("java.class.path")
- val request = new CreateInteractiveRequest()
- request.kind = kind
- request.conf = extraConf ++ Map(
- RSCConf.Entry.LIVY_JARS.key() -> "",
- RSCConf.Entry.CLIENT_IN_PROCESS.key() -> inProcess.toString,
- SparkLauncher.SPARK_MASTER -> "local",
- SparkLauncher.DRIVER_EXTRA_CLASSPATH -> classpath,
- SparkLauncher.EXECUTOR_EXTRA_CLASSPATH -> classpath
- )
- request
- }
-
-}
[08/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala b/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
deleted file mode 100644
index 7d7b94e..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
+++ /dev/null
@@ -1,609 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.interactive
-
-import java.io.{File, InputStream}
-import java.net.URI
-import java.nio.ByteBuffer
-import java.nio.file.{Files, Paths}
-import java.util.concurrent.TimeUnit
-import java.util.concurrent.atomic.AtomicLong
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-import scala.concurrent.Future
-import scala.concurrent.duration.{Duration, FiniteDuration}
-import scala.util.Random
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties
-import com.google.common.annotations.VisibleForTesting
-import org.apache.hadoop.fs.Path
-import org.apache.spark.launcher.SparkLauncher
-
-import com.cloudera.livy._
-import com.cloudera.livy.client.common.HttpMessages._
-import com.cloudera.livy.rsc.{PingJob, RSCClient, RSCConf}
-import com.cloudera.livy.rsc.driver.Statement
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions._
-import com.cloudera.livy.sessions.Session._
-import com.cloudera.livy.sessions.SessionState.Dead
-import com.cloudera.livy.util.LineBufferedProcess
-import com.cloudera.livy.utils.{AppInfo, LivySparkUtils, SparkApp, SparkAppListener}
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-case class InteractiveRecoveryMetadata(
- id: Int,
- appId: Option[String],
- appTag: String,
- kind: Kind,
- heartbeatTimeoutS: Int,
- owner: String,
- proxyUser: Option[String],
- rscDriverUri: Option[URI],
- version: Int = 1)
- extends RecoveryMetadata
-
-object InteractiveSession extends Logging {
- private[interactive] val SPARK_YARN_IS_PYTHON = "spark.yarn.isPython"
-
- val RECOVERY_SESSION_TYPE = "interactive"
-
- def create(
- id: Int,
- owner: String,
- proxyUser: Option[String],
- livyConf: LivyConf,
- request: CreateInteractiveRequest,
- sessionStore: SessionStore,
- mockApp: Option[SparkApp] = None,
- mockClient: Option[RSCClient] = None): InteractiveSession = {
- val appTag = s"livy-session-$id-${Random.alphanumeric.take(8).mkString}"
-
- val client = mockClient.orElse {
- val conf = SparkApp.prepareSparkConf(appTag, livyConf, prepareConf(
- request.conf, request.jars, request.files, request.archives, request.pyFiles, livyConf))
-
- val builderProperties = prepareBuilderProp(conf, request.kind, livyConf)
-
- val userOpts: Map[String, Option[String]] = Map(
- "spark.driver.cores" -> request.driverCores.map(_.toString),
- SparkLauncher.DRIVER_MEMORY -> request.driverMemory.map(_.toString),
- SparkLauncher.EXECUTOR_CORES -> request.executorCores.map(_.toString),
- SparkLauncher.EXECUTOR_MEMORY -> request.executorMemory.map(_.toString),
- "spark.executor.instances" -> request.numExecutors.map(_.toString),
- "spark.app.name" -> request.name.map(_.toString),
- "spark.yarn.queue" -> request.queue
- )
-
- userOpts.foreach { case (key, opt) =>
- opt.foreach { value => builderProperties.put(key, value) }
- }
-
- builderProperties.getOrElseUpdate("spark.app.name", s"livy-session-$id")
-
- info(s"Creating Interactive session $id: [owner: $owner, request: $request]")
- val builder = new LivyClientBuilder()
- .setAll(builderProperties.asJava)
- .setConf("livy.client.session-id", id.toString)
- .setConf(RSCConf.Entry.DRIVER_CLASS.key(), "com.cloudera.livy.repl.ReplDriver")
- .setConf(RSCConf.Entry.PROXY_USER.key(), proxyUser.orNull)
- .setURI(new URI("rsc:/"))
-
- Option(builder.build().asInstanceOf[RSCClient])
- }
-
- new InteractiveSession(
- id,
- None,
- appTag,
- client,
- SessionState.Starting(),
- request.kind,
- request.heartbeatTimeoutInSecond,
- livyConf,
- owner,
- proxyUser,
- sessionStore,
- mockApp)
- }
-
- def recover(
- metadata: InteractiveRecoveryMetadata,
- livyConf: LivyConf,
- sessionStore: SessionStore,
- mockApp: Option[SparkApp] = None,
- mockClient: Option[RSCClient] = None): InteractiveSession = {
- val client = mockClient.orElse(metadata.rscDriverUri.map { uri =>
- val builder = new LivyClientBuilder().setURI(uri)
- builder.build().asInstanceOf[RSCClient]
- })
-
- new InteractiveSession(
- metadata.id,
- metadata.appId,
- metadata.appTag,
- client,
- SessionState.Recovering(),
- metadata.kind,
- metadata.heartbeatTimeoutS,
- livyConf,
- metadata.owner,
- metadata.proxyUser,
- sessionStore,
- mockApp)
- }
-
- @VisibleForTesting
- private[interactive] def prepareBuilderProp(
- conf: Map[String, String],
- kind: Kind,
- livyConf: LivyConf): mutable.Map[String, String] = {
-
- val builderProperties = mutable.Map[String, String]()
- builderProperties ++= conf
-
- def livyJars(livyConf: LivyConf, scalaVersion: String): List[String] = {
- Option(livyConf.get(LivyConf.REPL_JARS)).map { jars =>
- val regex = """[\w-]+_(\d\.\d\d).*\.jar""".r
- jars.split(",").filter { name => new Path(name).getName match {
- // Filter out unmatched scala jars
- case regex(ver) => ver == scalaVersion
- // Keep all the java jars end with ".jar"
- case _ => name.endsWith(".jar")
- }
- }.toList
- }.getOrElse {
- val home = sys.env("LIVY_HOME")
- val jars = Option(new File(home, s"repl_$scalaVersion-jars"))
- .filter(_.isDirectory())
- .getOrElse(new File(home, s"repl/scala-$scalaVersion/target/jars"))
- require(jars.isDirectory(), "Cannot find Livy REPL jars.")
- jars.listFiles().map(_.getAbsolutePath()).toList
- }
- }
-
- def findSparkRArchive(): Option[String] = {
- Option(livyConf.get(RSCConf.Entry.SPARKR_PACKAGE.key())).orElse {
- sys.env.get("SPARK_HOME").map { case sparkHome =>
- val path = Seq(sparkHome, "R", "lib", "sparkr.zip").mkString(File.separator)
- val rArchivesFile = new File(path)
- require(rArchivesFile.exists(), "sparkr.zip not found; cannot run sparkr application.")
- rArchivesFile.getAbsolutePath()
- }
- }
- }
-
- def datanucleusJars(livyConf: LivyConf, sparkMajorVersion: Int): Seq[String] = {
- if (sys.env.getOrElse("LIVY_INTEGRATION_TEST", "false").toBoolean) {
- // datanucleus jars has already been in classpath in integration test
- Seq.empty
- } else {
- val sparkHome = livyConf.sparkHome().get
- val libdir = sparkMajorVersion match {
- case 1 =>
- if (new File(sparkHome, "RELEASE").isFile) {
- new File(sparkHome, "lib")
- } else {
- new File(sparkHome, "lib_managed/jars")
- }
- case 2 =>
- if (new File(sparkHome, "RELEASE").isFile) {
- new File(sparkHome, "jars")
- } else if (new File(sparkHome, "assembly/target/scala-2.11/jars").isDirectory) {
- new File(sparkHome, "assembly/target/scala-2.11/jars")
- } else {
- new File(sparkHome, "assembly/target/scala-2.10/jars")
- }
- case v =>
- throw new RuntimeException("Unsupported spark major version:" + sparkMajorVersion)
- }
- val jars = if (!libdir.isDirectory) {
- Seq.empty[String]
- } else {
- libdir.listFiles().filter(_.getName.startsWith("datanucleus-"))
- .map(_.getAbsolutePath).toSeq
- }
- if (jars.isEmpty) {
- warn("datanucleus jars can not be found")
- }
- jars
- }
- }
-
- /**
- * Look for hive-site.xml (for now just ignore spark.files defined in spark-defaults.conf)
- * 1. First look for hive-site.xml in user request
- * 2. Then look for that under classpath
- * @param livyConf
- * @return (hive-site.xml path, whether it is provided by user)
- */
- def hiveSiteFile(sparkFiles: Array[String], livyConf: LivyConf): (Option[File], Boolean) = {
- if (sparkFiles.exists(_.split("/").last == "hive-site.xml")) {
- (None, true)
- } else {
- val hiveSiteURL = getClass.getResource("/hive-site.xml")
- if (hiveSiteURL != null && hiveSiteURL.getProtocol == "file") {
- (Some(new File(hiveSiteURL.toURI)), false)
- } else {
- (None, false)
- }
- }
- }
-
- def findPySparkArchives(): Seq[String] = {
- Option(livyConf.get(RSCConf.Entry.PYSPARK_ARCHIVES))
- .map(_.split(",").toSeq)
- .getOrElse {
- sys.env.get("SPARK_HOME") .map { case sparkHome =>
- val pyLibPath = Seq(sparkHome, "python", "lib").mkString(File.separator)
- val pyArchivesFile = new File(pyLibPath, "pyspark.zip")
- require(pyArchivesFile.exists(),
- "pyspark.zip not found; cannot run pyspark application in YARN mode.")
-
- val py4jFile = Files.newDirectoryStream(Paths.get(pyLibPath), "py4j-*-src.zip")
- .iterator()
- .next()
- .toFile
-
- require(py4jFile.exists(),
- "py4j-*-src.zip not found; cannot run pyspark application in YARN mode.")
- Seq(pyArchivesFile.getAbsolutePath, py4jFile.getAbsolutePath)
- }.getOrElse(Seq())
- }
- }
-
- def mergeConfList(list: Seq[String], key: String): Unit = {
- if (list.nonEmpty) {
- builderProperties.get(key) match {
- case None =>
- builderProperties.put(key, list.mkString(","))
- case Some(oldList) =>
- val newList = (oldList :: list.toList).mkString(",")
- builderProperties.put(key, newList)
- }
- }
- }
-
- def mergeHiveSiteAndHiveDeps(sparkMajorVersion: Int): Unit = {
- val sparkFiles = conf.get("spark.files").map(_.split(",")).getOrElse(Array.empty[String])
- hiveSiteFile(sparkFiles, livyConf) match {
- case (_, true) =>
- debug("Enable HiveContext because hive-site.xml is found in user request.")
- mergeConfList(datanucleusJars(livyConf, sparkMajorVersion), LivyConf.SPARK_JARS)
- case (Some(file), false) =>
- debug("Enable HiveContext because hive-site.xml is found under classpath, "
- + file.getAbsolutePath)
- mergeConfList(List(file.getAbsolutePath), LivyConf.SPARK_FILES)
- mergeConfList(datanucleusJars(livyConf, sparkMajorVersion), LivyConf.SPARK_JARS)
- case (None, false) =>
- warn("Enable HiveContext but no hive-site.xml found under" +
- " classpath or user request.")
- }
- }
-
- kind match {
- case PySpark() | PySpark3() =>
- val pySparkFiles = if (!LivyConf.TEST_MODE) findPySparkArchives() else Nil
- mergeConfList(pySparkFiles, LivyConf.SPARK_PY_FILES)
- builderProperties.put(SPARK_YARN_IS_PYTHON, "true")
- case SparkR() =>
- val sparkRArchive = if (!LivyConf.TEST_MODE) findSparkRArchive() else None
- sparkRArchive.foreach { archive =>
- builderProperties.put(RSCConf.Entry.SPARKR_PACKAGE.key(), archive + "#sparkr")
- }
- case _ =>
- }
- builderProperties.put(RSCConf.Entry.SESSION_KIND.key, kind.toString)
-
- // Set Livy.rsc.jars from livy conf to rsc conf, RSC conf will take precedence if both are set.
- Option(livyConf.get(LivyConf.RSC_JARS)).foreach(
- builderProperties.getOrElseUpdate(RSCConf.Entry.LIVY_JARS.key(), _))
-
- require(livyConf.get(LivyConf.LIVY_SPARK_VERSION) != null)
- require(livyConf.get(LivyConf.LIVY_SPARK_SCALA_VERSION) != null)
-
- val (sparkMajorVersion, _) =
- LivySparkUtils.formatSparkVersion(livyConf.get(LivyConf.LIVY_SPARK_VERSION))
- val scalaVersion = livyConf.get(LivyConf.LIVY_SPARK_SCALA_VERSION)
-
- mergeConfList(livyJars(livyConf, scalaVersion), LivyConf.SPARK_JARS)
- val enableHiveContext = livyConf.getBoolean(LivyConf.ENABLE_HIVE_CONTEXT)
- // pass spark.livy.spark_major_version to driver
- builderProperties.put("spark.livy.spark_major_version", sparkMajorVersion.toString)
-
- if (sparkMajorVersion <= 1) {
- builderProperties.put("spark.repl.enableHiveContext",
- livyConf.getBoolean(LivyConf.ENABLE_HIVE_CONTEXT).toString)
- } else {
- val confVal = if (enableHiveContext) "hive" else "in-memory"
- builderProperties.put("spark.sql.catalogImplementation", confVal)
- }
-
- if (enableHiveContext) {
- mergeHiveSiteAndHiveDeps(sparkMajorVersion)
- }
-
- builderProperties
- }
-}
-
-class InteractiveSession(
- id: Int,
- appIdHint: Option[String],
- appTag: String,
- client: Option[RSCClient],
- initialState: SessionState,
- val kind: Kind,
- heartbeatTimeoutS: Int,
- livyConf: LivyConf,
- owner: String,
- override val proxyUser: Option[String],
- sessionStore: SessionStore,
- mockApp: Option[SparkApp]) // For unit test.
- extends Session(id, owner, livyConf)
- with SessionHeartbeat
- with SparkAppListener {
-
- import InteractiveSession._
-
- private var serverSideState: SessionState = initialState
-
- override protected val heartbeatTimeout: FiniteDuration = {
- val heartbeatTimeoutInSecond = heartbeatTimeoutS
- Duration(heartbeatTimeoutInSecond, TimeUnit.SECONDS)
- }
- private val operations = mutable.Map[Long, String]()
- private val operationCounter = new AtomicLong(0)
- private var rscDriverUri: Option[URI] = None
- private var sessionLog: IndexedSeq[String] = IndexedSeq.empty
- private val sessionSaveLock = new Object()
-
- _appId = appIdHint
- sessionStore.save(RECOVERY_SESSION_TYPE, recoveryMetadata)
- heartbeat()
-
- private val app = mockApp.orElse {
- if (livyConf.isRunningOnYarn()) {
- val driverProcess = client.flatMap { c => Option(c.getDriverProcess) }
- .map(new LineBufferedProcess(_))
- // When Livy is running with YARN, SparkYarnApp can provide better YARN integration.
- // (e.g. Reflect YARN application state to session state).
- Option(SparkApp.create(appTag, appId, driverProcess, livyConf, Some(this)))
- } else {
- // When Livy is running with other cluster manager, SparkApp doesn't provide any
- // additional benefit over controlling RSCDriver using RSCClient. Don't use it.
- None
- }
- }
-
- if (client.isEmpty) {
- transition(Dead())
- val msg = s"Cannot recover interactive session $id because its RSCDriver URI is unknown."
- info(msg)
- sessionLog = IndexedSeq(msg)
- } else {
- val uriFuture = Future { client.get.getServerUri.get() }
-
- uriFuture onSuccess { case url =>
- rscDriverUri = Option(url)
- sessionSaveLock.synchronized {
- sessionStore.save(RECOVERY_SESSION_TYPE, recoveryMetadata)
- }
- }
- uriFuture onFailure { case e => warn("Fail to get rsc uri", e) }
-
- // Send a dummy job that will return once the client is ready to be used, and set the
- // state to "idle" at that point.
- client.get.submit(new PingJob()).addListener(new JobHandle.Listener[Void]() {
- override def onJobQueued(job: JobHandle[Void]): Unit = { }
- override def onJobStarted(job: JobHandle[Void]): Unit = { }
-
- override def onJobCancelled(job: JobHandle[Void]): Unit = errorOut()
-
- override def onJobFailed(job: JobHandle[Void], cause: Throwable): Unit = errorOut()
-
- override def onJobSucceeded(job: JobHandle[Void], result: Void): Unit = {
- transition(SessionState.Running())
- info(s"Interactive session $id created [appid: ${appId.orNull}, owner: $owner, proxyUser:" +
- s" $proxyUser, state: ${state.toString}, kind: ${kind.toString}, " +
- s"info: ${appInfo.asJavaMap}]")
- }
-
- private def errorOut(): Unit = {
- // Other code might call stop() to close the RPC channel. When RPC channel is closing,
- // this callback might be triggered. Check and don't call stop() to avoid nested called
- // if the session is already shutting down.
- if (serverSideState != SessionState.ShuttingDown()) {
- transition(SessionState.Error())
- stop()
- app.foreach { a =>
- info(s"Failed to ping RSC driver for session $id. Killing application.")
- a.kill()
- }
- }
- }
- })
- }
-
- override def logLines(): IndexedSeq[String] = app.map(_.log()).getOrElse(sessionLog)
-
- override def recoveryMetadata: RecoveryMetadata =
- InteractiveRecoveryMetadata(
- id, appId, appTag, kind, heartbeatTimeout.toSeconds.toInt, owner, proxyUser, rscDriverUri)
-
- override def state: SessionState = {
- if (serverSideState.isInstanceOf[SessionState.Running]) {
- // If session is in running state, return the repl state from RSCClient.
- client
- .flatMap(s => Option(s.getReplState))
- .map(SessionState(_))
- .getOrElse(SessionState.Busy()) // If repl state is unknown, assume repl is busy.
- } else {
- serverSideState
- }
- }
-
- override def stopSession(): Unit = {
- try {
- transition(SessionState.ShuttingDown())
- sessionStore.remove(RECOVERY_SESSION_TYPE, id)
- client.foreach { _.stop(true) }
- } catch {
- case _: Exception =>
- app.foreach {
- warn(s"Failed to stop RSCDriver. Killing it...")
- _.kill()
- }
- } finally {
- transition(SessionState.Dead())
- }
- }
-
- def statements: IndexedSeq[Statement] = {
- ensureActive()
- val r = client.get.getReplJobResults().get()
- r.statements.toIndexedSeq
- }
-
- def getStatement(stmtId: Int): Option[Statement] = {
- ensureActive()
- val r = client.get.getReplJobResults(stmtId, 1).get()
- if (r.statements.length < 1) {
- None
- } else {
- Option(r.statements(0))
- }
- }
-
- def interrupt(): Future[Unit] = {
- stop()
- }
-
- def executeStatement(content: ExecuteRequest): Statement = {
- ensureRunning()
- recordActivity()
-
- val id = client.get.submitReplCode(content.code).get
- client.get.getReplJobResults(id, 1).get().statements(0)
- }
-
- def cancelStatement(statementId: Int): Unit = {
- ensureRunning()
- recordActivity()
- client.get.cancelReplCode(statementId)
- }
-
- def runJob(job: Array[Byte]): Long = {
- performOperation(job, true)
- }
-
- def submitJob(job: Array[Byte]): Long = {
- performOperation(job, false)
- }
-
- def addFile(fileStream: InputStream, fileName: String): Unit = {
- addFile(copyResourceToHDFS(fileStream, fileName))
- }
-
- def addJar(jarStream: InputStream, jarName: String): Unit = {
- addJar(copyResourceToHDFS(jarStream, jarName))
- }
-
- def addFile(uri: URI): Unit = {
- ensureActive()
- recordActivity()
- client.get.addFile(resolveURI(uri, livyConf)).get()
- }
-
- def addJar(uri: URI): Unit = {
- ensureActive()
- recordActivity()
- client.get.addJar(resolveURI(uri, livyConf)).get()
- }
-
- def jobStatus(id: Long): Any = {
- ensureActive()
- val clientJobId = operations(id)
- recordActivity()
- // TODO: don't block indefinitely?
- val status = client.get.getBypassJobStatus(clientJobId).get()
- new JobStatus(id, status.state, status.result, status.error)
- }
-
- def cancelJob(id: Long): Unit = {
- ensureActive()
- recordActivity()
- operations.remove(id).foreach { client.get.cancel }
- }
-
- private def transition(newState: SessionState) = synchronized {
- // When a statement returns an error, the session should transit to error state.
- // If the session crashed because of the error, the session should instead go to dead state.
- // Since these 2 transitions are triggered by different threads, there's a race condition.
- // Make sure we won't transit from dead to error state.
- val areSameStates = serverSideState.getClass() == newState.getClass()
- val transitFromInactiveToActive = !serverSideState.isActive && newState.isActive
- if (!areSameStates && !transitFromInactiveToActive) {
- debug(s"$this session state change from ${serverSideState} to $newState")
- serverSideState = newState
- }
- }
-
- private def ensureActive(): Unit = synchronized {
- require(serverSideState.isActive, "Session isn't active.")
- require(client.isDefined, "Session is active but client hasn't been created.")
- }
-
- private def ensureRunning(): Unit = synchronized {
- serverSideState match {
- case SessionState.Running() =>
- case _ =>
- throw new IllegalStateException("Session is in state %s" format serverSideState)
- }
- }
-
- private def performOperation(job: Array[Byte], sync: Boolean): Long = {
- ensureActive()
- recordActivity()
- val future = client.get.bypass(ByteBuffer.wrap(job), sync)
- val opId = operationCounter.incrementAndGet()
- operations(opId) = future
- opId
- }
-
- override def appIdKnown(appId: String): Unit = {
- _appId = Option(appId)
- sessionSaveLock.synchronized {
- sessionStore.save(RECOVERY_SESSION_TYPE, recoveryMetadata)
- }
- }
-
- override def stateChanged(oldState: SparkApp.State, newState: SparkApp.State): Unit = {
- synchronized {
- debug(s"$this app state changed from $oldState to $newState")
- newState match {
- case SparkApp.State.FINISHED | SparkApp.State.KILLED | SparkApp.State.FAILED =>
- transition(SessionState.Dead())
- case _ =>
- }
- }
- }
-
- override def infoChanged(appInfo: AppInfo): Unit = { this.appInfo = appInfo }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSessionServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSessionServlet.scala b/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSessionServlet.scala
deleted file mode 100644
index c2263db..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSessionServlet.scala
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.interactive
-
-import java.net.URI
-import javax.servlet.http.HttpServletRequest
-
-import scala.collection.JavaConverters._
-import scala.concurrent._
-import scala.concurrent.duration._
-
-import org.json4s.jackson.Json4sScalaModule
-import org.scalatra._
-import org.scalatra.servlet.FileUploadSupport
-
-import com.cloudera.livy.{ExecuteRequest, JobHandle, LivyConf, Logging}
-import com.cloudera.livy.client.common.HttpMessages
-import com.cloudera.livy.client.common.HttpMessages._
-import com.cloudera.livy.rsc.driver.Statement
-import com.cloudera.livy.server.SessionServlet
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions._
-
-object InteractiveSessionServlet extends Logging
-
-class InteractiveSessionServlet(
- sessionManager: InteractiveSessionManager,
- sessionStore: SessionStore,
- livyConf: LivyConf)
- extends SessionServlet(sessionManager, livyConf)
- with SessionHeartbeatNotifier[InteractiveSession, InteractiveRecoveryMetadata]
- with FileUploadSupport
-{
-
- mapper.registerModule(new SessionKindModule())
- .registerModule(new Json4sScalaModule())
-
- override protected def createSession(req: HttpServletRequest): InteractiveSession = {
- val createRequest = bodyAs[CreateInteractiveRequest](req)
- val proxyUser = checkImpersonation(createRequest.proxyUser, req)
- InteractiveSession.create(
- sessionManager.nextId(),
- remoteUser(req),
- proxyUser,
- livyConf,
- createRequest,
- sessionStore)
- }
-
- override protected[interactive] def clientSessionView(
- session: InteractiveSession,
- req: HttpServletRequest): Any = {
- val logs =
- if (hasAccess(session.owner, req)) {
- Option(session.logLines())
- .map { lines =>
- val size = 10
- var from = math.max(0, lines.length - size)
- val until = from + size
-
- lines.view(from, until)
- }
- .getOrElse(Nil)
- } else {
- Nil
- }
-
- new SessionInfo(session.id, session.appId.orNull, session.owner, session.proxyUser.orNull,
- session.state.toString, session.kind.toString, session.appInfo.asJavaMap, logs.asJava)
- }
-
- post("/:id/stop") {
- withSession { session =>
- Await.ready(session.stop(), Duration.Inf)
- NoContent()
- }
- }
-
- post("/:id/interrupt") {
- withSession { session =>
- Await.ready(session.interrupt(), Duration.Inf)
- Ok(Map("msg" -> "interrupted"))
- }
- }
-
- get("/:id/statements") {
- withSession { session =>
- val statements = session.statements
- val from = params.get("from").map(_.toInt).getOrElse(0)
- val size = params.get("size").map(_.toInt).getOrElse(statements.length)
-
- Map(
- "total_statements" -> statements.length,
- "statements" -> statements.view(from, from + size)
- )
- }
- }
-
- val getStatement = get("/:id/statements/:statementId") {
- withSession { session =>
- val statementId = params("statementId").toInt
-
- session.getStatement(statementId).getOrElse(NotFound("Statement not found"))
- }
- }
-
- jpost[ExecuteRequest]("/:id/statements") { req =>
- withSession { session =>
- val statement = session.executeStatement(req)
-
- Created(statement,
- headers = Map(
- "Location" -> url(getStatement,
- "id" -> session.id.toString,
- "statementId" -> statement.id.toString)))
- }
- }
-
- post("/:id/statements/:statementId/cancel") {
- withSession { session =>
- val statementId = params("statementId")
- session.cancelStatement(statementId.toInt)
- Ok(Map("msg" -> "canceled"))
- }
- }
- // This endpoint is used by the client-http module to "connect" to an existing session and
- // update its last activity time. It performs authorization checks to make sure the caller
- // has access to the session, so even though it returns the same data, it behaves differently
- // from get("/:id").
- post("/:id/connect") {
- withSession { session =>
- session.recordActivity()
- Ok(clientSessionView(session, request))
- }
- }
-
- jpost[SerializedJob]("/:id/submit-job") { req =>
- withSession { session =>
- try {
- require(req.job != null && req.job.length > 0, "no job provided.")
- val jobId = session.submitJob(req.job)
- Created(new JobStatus(jobId, JobHandle.State.SENT, null, null))
- } catch {
- case e: Throwable =>
- e.printStackTrace()
- throw e
- }
- }
- }
-
- jpost[SerializedJob]("/:id/run-job") { req =>
- withSession { session =>
- require(req.job != null && req.job.length > 0, "no job provided.")
- val jobId = session.runJob(req.job)
- Created(new JobStatus(jobId, JobHandle.State.SENT, null, null))
- }
- }
-
- post("/:id/upload-jar") {
- withSession { lsession =>
- fileParams.get("jar") match {
- case Some(file) =>
- lsession.addJar(file.getInputStream, file.name)
- case None =>
- BadRequest("No jar sent!")
- }
- }
- }
-
- post("/:id/upload-pyfile") {
- withSession { lsession =>
- fileParams.get("file") match {
- case Some(file) =>
- lsession.addJar(file.getInputStream, file.name)
- case None =>
- BadRequest("No file sent!")
- }
- }
- }
-
- post("/:id/upload-file") {
- withSession { lsession =>
- fileParams.get("file") match {
- case Some(file) =>
- lsession.addFile(file.getInputStream, file.name)
- case None =>
- BadRequest("No file sent!")
- }
- }
- }
-
- jpost[AddResource]("/:id/add-jar") { req =>
- withSession { lsession =>
- addJarOrPyFile(req, lsession)
- }
- }
-
- jpost[AddResource]("/:id/add-pyfile") { req =>
- withSession { lsession =>
- lsession.kind match {
- case PySpark() | PySpark3() => addJarOrPyFile(req, lsession)
- case _ => BadRequest("Only supported for pyspark sessions.")
- }
- }
- }
-
- jpost[AddResource]("/:id/add-file") { req =>
- withSession { lsession =>
- val uri = new URI(req.uri)
- lsession.addFile(uri)
- }
- }
-
- get("/:id/jobs/:jobid") {
- withSession { lsession =>
- val jobId = params("jobid").toLong
- Ok(lsession.jobStatus(jobId))
- }
- }
-
- post("/:id/jobs/:jobid/cancel") {
- withSession { lsession =>
- val jobId = params("jobid").toLong
- lsession.cancelJob(jobId)
- }
- }
-
- private def addJarOrPyFile(req: HttpMessages.AddResource, session: InteractiveSession): Unit = {
- val uri = new URI(req.uri)
- session.addJar(uri)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/interactive/SessionHeartbeat.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/interactive/SessionHeartbeat.scala b/server/src/main/scala/com/cloudera/livy/server/interactive/SessionHeartbeat.scala
deleted file mode 100644
index 20bc582..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/interactive/SessionHeartbeat.scala
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.server.interactive
-
-import java.util.Date
-
-import scala.concurrent.duration.{Deadline, Duration, FiniteDuration}
-
-import com.cloudera.livy.sessions.Session.RecoveryMetadata
-import com.cloudera.livy.LivyConf
-import com.cloudera.livy.server.SessionServlet
-import com.cloudera.livy.sessions.{Session, SessionManager}
-
-/**
- * A session trait to provide heartbeat expiration check.
- * Note: Session will not expire if heartbeat() was never called.
- */
-trait SessionHeartbeat {
- protected val heartbeatTimeout: FiniteDuration
-
- private var _lastHeartbeat: Date = _ // For reporting purpose
- private var heartbeatDeadline: Option[Deadline] = None
-
- def heartbeat(): Unit = synchronized {
- if (heartbeatTimeout > Duration.Zero) {
- heartbeatDeadline = Some(heartbeatTimeout.fromNow)
- }
-
- _lastHeartbeat = new Date()
- }
-
- def lastHeartbeat: Date = synchronized { _lastHeartbeat }
-
- def heartbeatExpired: Boolean = synchronized { heartbeatDeadline.exists(_.isOverdue()) }
-}
-
-/**
- * Servlet can mixin this trait to update session's heartbeat
- * whenever a /sessions/:id REST call is made. e.g. GET /sessions/:id
- * Note: GET /sessions doesn't update heartbeats.
- */
-trait SessionHeartbeatNotifier[S <: Session with SessionHeartbeat, R <: RecoveryMetadata]
- extends SessionServlet[S, R] {
-
- abstract override protected def withUnprotectedSession(fn: (S => Any)): Any = {
- super.withUnprotectedSession { s =>
- s.heartbeat()
- fn(s)
- }
- }
-
- abstract override protected def withSession(fn: (S => Any)): Any = {
- super.withSession { s =>
- s.heartbeat()
- fn(s)
- }
- }
-}
-
-/**
- * A SessionManager trait.
- * It will create a thread that periodically deletes sessions with expired heartbeat.
- */
-trait SessionHeartbeatWatchdog[S <: Session with SessionHeartbeat, R <: RecoveryMetadata] {
- self: SessionManager[S, R] =>
-
- private val watchdogThread = new Thread(s"HeartbeatWatchdog-${self.getClass.getName}") {
- override def run(): Unit = {
- val interval = livyConf.getTimeAsMs(LivyConf.HEARTBEAT_WATCHDOG_INTERVAL)
- info("Heartbeat watchdog thread started.")
- while (true) {
- deleteExpiredSessions()
- Thread.sleep(interval)
- }
- }
- }
-
- protected def start(): Unit = {
- assert(!watchdogThread.isAlive())
-
- watchdogThread.setDaemon(true)
- watchdogThread.start()
- }
-
- private[interactive] def deleteExpiredSessions(): Unit = {
- // Delete takes time. If we use .filter().foreach() here, the time difference between we check
- // expiration and the time we delete the session might be huge. To avoid that, check expiration
- // inside the foreach block.
- sessions.values.foreach { s =>
- if (s.heartbeatExpired) {
- info(s"Session ${s.id} expired. Last heartbeat is at ${s.lastHeartbeat}.")
- try { delete(s) } catch {
- case t: Throwable =>
- warn(s"Exception was thrown when deleting expired session ${s.id}", t)
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/recovery/BlackholeStateStore.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/recovery/BlackholeStateStore.scala b/server/src/main/scala/com/cloudera/livy/server/recovery/BlackholeStateStore.scala
deleted file mode 100644
index 89b133e..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/recovery/BlackholeStateStore.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.recovery
-
-import scala.reflect.ClassTag
-
-import com.cloudera.livy.LivyConf
-
-/**
- * This is a blackhole implementation of StateStore.
- * Livy will use this when session recovery is disabled.
- */
-class BlackholeStateStore(livyConf: LivyConf) extends StateStore(livyConf) {
- def set(key: String, value: Object): Unit = {}
-
- def get[T: ClassTag](key: String): Option[T] = None
-
- def getChildren(key: String): Seq[String] = List.empty[String]
-
- def remove(key: String): Unit = {}
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/recovery/FileSystemStateStore.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/recovery/FileSystemStateStore.scala b/server/src/main/scala/com/cloudera/livy/server/recovery/FileSystemStateStore.scala
deleted file mode 100644
index 5e17678..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/recovery/FileSystemStateStore.scala
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.recovery
-
-import java.io.{FileNotFoundException, IOException}
-import java.net.URI
-import java.util
-
-import scala.reflect.ClassTag
-import scala.util.control.NonFatal
-
-import org.apache.commons.io.IOUtils
-import org.apache.hadoop.fs._
-import org.apache.hadoop.fs.Options.{CreateOpts, Rename}
-import org.apache.hadoop.fs.permission.{FsAction, FsPermission}
-
-import com.cloudera.livy.{LivyConf, Logging}
-import com.cloudera.livy.Utils.usingResource
-
-class FileSystemStateStore(
- livyConf: LivyConf,
- mockFileContext: Option[FileContext])
- extends StateStore(livyConf) with Logging {
-
- // Constructor defined for StateStore factory to new this class using reflection.
- def this(livyConf: LivyConf) {
- this(livyConf, None)
- }
-
- private val fsUri = {
- val fsPath = livyConf.get(LivyConf.RECOVERY_STATE_STORE_URL)
- require(!fsPath.isEmpty, s"Please config ${LivyConf.RECOVERY_STATE_STORE_URL.key}.")
- new URI(fsPath)
- }
-
- private val fileContext: FileContext = mockFileContext.getOrElse {
- FileContext.getFileContext(fsUri)
- }
-
- {
- // Only Livy user should have access to state files.
- fileContext.setUMask(new FsPermission("077"))
-
- // Create state store dir if it doesn't exist.
- val stateStorePath = absPath(".")
- try {
- fileContext.mkdir(stateStorePath, FsPermission.getDirDefault(), true)
- } catch {
- case _: FileAlreadyExistsException =>
- if (!fileContext.getFileStatus(stateStorePath).isDirectory()) {
- throw new IOException(s"$stateStorePath is not a directory.")
- }
- }
-
- // Check permission of state store dir.
- val fileStatus = fileContext.getFileStatus(absPath("."))
- require(fileStatus.getPermission.getUserAction() == FsAction.ALL,
- s"Livy doesn't have permission to access state store: $fsUri.")
- if (fileStatus.getPermission.getGroupAction != FsAction.NONE) {
- warn(s"Group users have permission to access state store: $fsUri. This is insecure.")
- }
- if (fileStatus.getPermission.getOtherAction != FsAction.NONE) {
- warn(s"Other users have permission to access state store: $fsUri. This is in secure.")
- }
- }
-
- override def set(key: String, value: Object): Unit = {
- // Write to a temp file then rename to avoid file corruption if livy-server crashes
- // in the middle of the write.
- val tmpPath = absPath(s"$key.tmp")
- val createFlag = util.EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
-
- usingResource(fileContext.create(tmpPath, createFlag, CreateOpts.createParent())) { tmpFile =>
- tmpFile.write(serializeToBytes(value))
- tmpFile.close()
- // Assume rename is atomic.
- fileContext.rename(tmpPath, absPath(key), Rename.OVERWRITE)
- }
-
- try {
- val crcPath = new Path(tmpPath.getParent, s".${tmpPath.getName}.crc")
- fileContext.delete(crcPath, false)
- } catch {
- case NonFatal(e) => // Swallow the exception.
- }
- }
-
- override def get[T: ClassTag](key: String): Option[T] = {
- try {
- usingResource(fileContext.open(absPath(key))) { is =>
- Option(deserialize[T](IOUtils.toByteArray(is)))
- }
- } catch {
- case _: FileNotFoundException => None
- case e: IOException =>
- warn(s"Failed to read $key from state store.", e)
- None
- }
- }
-
- override def getChildren(key: String): Seq[String] = {
- try {
- fileContext.util.listStatus(absPath(key)).map(_.getPath.getName)
- } catch {
- case _: FileNotFoundException => Seq.empty
- case e: IOException =>
- warn(s"Failed to list $key from state store.", e)
- Seq.empty
- }
- }
-
- override def remove(key: String): Unit = {
- fileContext.delete(absPath(key), false)
- }
-
- private def absPath(key: String): Path = new Path(fsUri.getPath(), key)
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/recovery/SessionStore.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/recovery/SessionStore.scala b/server/src/main/scala/com/cloudera/livy/server/recovery/SessionStore.scala
deleted file mode 100644
index b7a178c..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/recovery/SessionStore.scala
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.recovery
-
-import java.io.IOException
-
-import scala.reflect.ClassTag
-import scala.util.{Failure, Success, Try}
-import scala.util.control.NonFatal
-
-import com.cloudera.livy.{LivyConf, Logging}
-import com.cloudera.livy.sessions.Session.RecoveryMetadata
-
-private[recovery] case class SessionManagerState(nextSessionId: Int)
-
-/**
- * SessionStore provides high level functions to get/save session state from/to StateStore.
- */
-class SessionStore(
- livyConf: LivyConf,
- store: => StateStore = StateStore.get) // For unit testing.
- extends Logging {
-
- private val STORE_VERSION: String = "v1"
-
- /**
- * Persist a session to the session state store.
- * @param m RecoveryMetadata for the session.
- */
- def save(sessionType: String, m: RecoveryMetadata): Unit = {
- store.set(sessionPath(sessionType, m.id), m)
- }
-
- def saveNextSessionId(sessionType: String, id: Int): Unit = {
- store.set(sessionManagerPath(sessionType), SessionManagerState(id))
- }
-
- /**
- * Return all sessions stored in the store with specified session type.
- */
- def getAllSessions[T <: RecoveryMetadata : ClassTag](sessionType: String): Seq[Try[T]] = {
- store.getChildren(sessionPath(sessionType))
- .flatMap { c => Try(c.toInt).toOption } // Ignore all non numerical keys
- .flatMap { id =>
- val p = sessionPath(sessionType, id)
- try {
- store.get[T](p).map(Success(_))
- } catch {
- case NonFatal(e) => Some(Failure(new IOException(s"Error getting session $p", e)))
- }
- }
- }
-
- /**
- * Return the next unused session id with specified session type.
- * If checks the SessionManagerState stored and returns the next free session id.
- * If no SessionManagerState is stored, it returns 0.
- *
- * @throws Exception If SessionManagerState stored is corrupted, it throws an error.
- */
- def getNextSessionId(sessionType: String): Int = {
- store.get[SessionManagerState](sessionManagerPath(sessionType))
- .map(_.nextSessionId).getOrElse(0)
- }
-
- /**
- * Remove a session from the state store.
- */
- def remove(sessionType: String, id: Int): Unit = {
- store.remove(sessionPath(sessionType, id))
- }
-
- private def sessionManagerPath(sessionType: String): String =
- s"$STORE_VERSION/$sessionType/state"
-
- private def sessionPath(sessionType: String): String =
- s"$STORE_VERSION/$sessionType"
-
- private def sessionPath(sessionType: String, id: Int): String =
- s"$STORE_VERSION/$sessionType/$id"
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/recovery/StateStore.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/recovery/StateStore.scala b/server/src/main/scala/com/cloudera/livy/server/recovery/StateStore.scala
deleted file mode 100644
index 25eb238..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/recovery/StateStore.scala
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.server.recovery
-
-import scala.reflect.{classTag, ClassTag}
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.module.scala.DefaultScalaModule
-
-import com.cloudera.livy.{LivyConf, Logging}
-import com.cloudera.livy.sessions.SessionKindModule
-import com.cloudera.livy.sessions.SessionManager._
-
-protected trait JsonMapper {
- protected val mapper = new ObjectMapper()
- .registerModule(DefaultScalaModule)
- .registerModule(new SessionKindModule())
-
- def serializeToBytes(value: Object): Array[Byte] = mapper.writeValueAsBytes(value)
-
- def deserialize[T: ClassTag](json: Array[Byte]): T =
- mapper.readValue(json, classTag[T].runtimeClass.asInstanceOf[Class[T]])
-}
-
-/**
- * Interface of a key-value pair storage for state storage.
- * It's responsible for de/serialization and retrieving/storing object.
- * It's the low level interface used by higher level classes like SessionStore.
- *
- * Hardcoded to use JSON serialization for now for easier ops. Will add better serialization later.
- */
-abstract class StateStore(livyConf: LivyConf) extends JsonMapper {
- /**
- * Set a key-value pair to this state store. It overwrites existing value.
- * @throws Exception Throw when persisting the state store fails.
- */
- def set(key: String, value: Object): Unit
-
- /**
- * Get a key-value pair from this state store.
- * @return Value if the key exists. None if the key doesn't exist.
- * @throws Exception Throw when deserialization of the stored value fails.
- */
- def get[T: ClassTag](key: String): Option[T]
-
- /**
- * Treat keys in this state store as a directory tree and
- * return names of the direct children of the key.
- * @return List of names of the direct children of the key.
- * Empty list if the key doesn't exist or have no child.
- */
- def getChildren(key: String): Seq[String]
-
- /**
- * Remove the key from this state store. Does not throw if the key doesn't exist.
- * @throws Exception Throw when persisting the state store fails.
- */
- def remove(key: String): Unit
-}
-
-/**
- * Factory to create the store chosen in LivyConf.
- */
-object StateStore extends Logging {
- private[this] var stateStore: Option[StateStore] = None
-
- def init(livyConf: LivyConf): Unit = synchronized {
- if (stateStore.isEmpty) {
- val fileStateStoreClassTag = pickStateStore(livyConf)
- stateStore = Option(fileStateStoreClassTag.getDeclaredConstructor(classOf[LivyConf])
- .newInstance(livyConf).asInstanceOf[StateStore])
- info(s"Using ${stateStore.get.getClass.getSimpleName} for recovery.")
- }
- }
-
- def cleanup(): Unit = synchronized {
- stateStore = None
- }
-
- def get: StateStore = {
- assert(stateStore.isDefined, "StateStore hasn't been initialized.")
- stateStore.get
- }
-
- private[recovery] def pickStateStore(livyConf: LivyConf): Class[_] = {
- livyConf.get(LivyConf.RECOVERY_MODE) match {
- case SESSION_RECOVERY_MODE_OFF => classOf[BlackholeStateStore]
- case SESSION_RECOVERY_MODE_RECOVERY =>
- livyConf.get(LivyConf.RECOVERY_STATE_STORE) match {
- case "filesystem" => classOf[FileSystemStateStore]
- case "zookeeper" => classOf[ZooKeeperStateStore]
- case ss => throw new IllegalArgumentException(s"Unsupported state store $ss")
- }
- case rm => throw new IllegalArgumentException(s"Unsupported recovery mode $rm")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/recovery/ZooKeeperStateStore.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/recovery/ZooKeeperStateStore.scala b/server/src/main/scala/com/cloudera/livy/server/recovery/ZooKeeperStateStore.scala
deleted file mode 100644
index 753ea22..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/recovery/ZooKeeperStateStore.scala
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.server.recovery
-
-import scala.collection.JavaConverters._
-import scala.reflect.ClassTag
-import scala.util.Try
-
-import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
-import org.apache.curator.framework.api.UnhandledErrorListener
-import org.apache.curator.retry.RetryNTimes
-import org.apache.zookeeper.KeeperException.NoNodeException
-
-import com.cloudera.livy.{LivyConf, Logging}
-import com.cloudera.livy.LivyConf.Entry
-
-object ZooKeeperStateStore {
- val ZK_KEY_PREFIX_CONF = Entry("livy.server.recovery.zk-state-store.key-prefix", "livy")
- val ZK_RETRY_CONF = Entry("livy.server.recovery.zk-state-store.retry-policy", "5,100")
-}
-
-class ZooKeeperStateStore(
- livyConf: LivyConf,
- mockCuratorClient: Option[CuratorFramework] = None) // For testing
- extends StateStore(livyConf) with Logging {
-
- import ZooKeeperStateStore._
-
- // Constructor defined for StateStore factory to new this class using reflection.
- def this(livyConf: LivyConf) {
- this(livyConf, None)
- }
-
- private val zkAddress = livyConf.get(LivyConf.RECOVERY_STATE_STORE_URL)
- require(!zkAddress.isEmpty, s"Please config ${LivyConf.RECOVERY_STATE_STORE_URL.key}.")
- private val zkKeyPrefix = livyConf.get(ZK_KEY_PREFIX_CONF)
- private val retryValue = livyConf.get(ZK_RETRY_CONF)
- // a regex to match patterns like "m, n" where m and n both are integer values
- private val retryPattern = """\s*(\d+)\s*,\s*(\d+)\s*""".r
- private[recovery] val retryPolicy = retryValue match {
- case retryPattern(n, sleepMs) => new RetryNTimes(n.toInt, sleepMs.toInt)
- case _ => throw new IllegalArgumentException(
- s"$ZK_KEY_PREFIX_CONF contains bad value: $retryValue. " +
- "Correct format is <max retry count>,<sleep ms between retry>. e.g. 5,100")
- }
-
- private val curatorClient = mockCuratorClient.getOrElse {
- CuratorFrameworkFactory.newClient(zkAddress, retryPolicy)
- }
-
- Runtime.getRuntime.addShutdownHook(new Thread(new Runnable {
- override def run(): Unit = {
- curatorClient.close()
- }
- }))
-
- curatorClient.getUnhandledErrorListenable().addListener(new UnhandledErrorListener {
- def unhandledError(message: String, e: Throwable): Unit = {
- error(s"Fatal Zookeeper error. Shutting down Livy server.")
- System.exit(1)
- }
- })
- curatorClient.start()
- // TODO Make sure ZK path has proper secure permissions so that other users cannot read its
- // contents.
-
- override def set(key: String, value: Object): Unit = {
- val prefixedKey = prefixKey(key)
- val data = serializeToBytes(value)
- if (curatorClient.checkExists().forPath(prefixedKey) == null) {
- curatorClient.create().creatingParentsIfNeeded().forPath(prefixedKey, data)
- } else {
- curatorClient.setData().forPath(prefixedKey, data)
- }
- }
-
- override def get[T: ClassTag](key: String): Option[T] = {
- val prefixedKey = prefixKey(key)
- if (curatorClient.checkExists().forPath(prefixedKey) == null) {
- None
- } else {
- Option(deserialize[T](curatorClient.getData().forPath(prefixedKey)))
- }
- }
-
- override def getChildren(key: String): Seq[String] = {
- val prefixedKey = prefixKey(key)
- if (curatorClient.checkExists().forPath(prefixedKey) == null) {
- Seq.empty[String]
- } else {
- curatorClient.getChildren.forPath(prefixedKey).asScala
- }
- }
-
- override def remove(key: String): Unit = {
- try {
- curatorClient.delete().guaranteed().forPath(prefixKey(key))
- } catch {
- case _: NoNodeException =>
- }
- }
-
- private def prefixKey(key: String) = s"/$zkKeyPrefix/$key"
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/server/ui/UIServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/ui/UIServlet.scala b/server/src/main/scala/com/cloudera/livy/server/ui/UIServlet.scala
deleted file mode 100644
index 26c5e26..0000000
--- a/server/src/main/scala/com/cloudera/livy/server/ui/UIServlet.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.ui
-
-import scala.xml.Node
-
-import org.scalatra.ScalatraServlet
-
-import com.cloudera.livy.LivyConf
-
-class UIServlet extends ScalatraServlet {
- before() { contentType = "text/html" }
-
- def getHeader(title: String): Seq[Node] =
- <head>
- <link rel="stylesheet" href="/static/bootstrap.min.css" type="text/css"/>
- <link rel="stylesheet" href="/static/dataTables.bootstrap.min.css" type="text/css"/>
- <link rel="stylesheet" href="/static/livy-ui.css" type="text/css"/>
- <script src="/static/jquery-3.2.1.min.js"></script>
- <script src="/static/bootstrap.min.js"></script>
- <script src="/static/jquery.dataTables.min.js"></script>
- <script src="/static/dataTables.bootstrap.min.js"></script>
- <script src="/static/all-sessions.js"></script>
- <title>{title}</title>
- </head>
-
- def navBar(pageName: String): Seq[Node] =
- <nav class="navbar navbar-default">
- <div class="container-fluid">
- <div class="navbar-header">
- <a class="navbar-brand" href="#">
- <img alt="Livy" src="/static/livy-mini-logo.png"/>
- </a>
- </div>
- <div class="collapse navbar-collapse">
- <ul class="nav navbar-nav">
- <li><a href="#">{pageName}</a></li>
- </ul>
- </div>
- </div>
- </nav>
-
- def createPage(pageName: String, pageContents: Seq[Node]): Seq[Node] =
- <html>
- {getHeader("Livy - " + pageName)}
- <body>
- <div class="container">
- {navBar(pageName)}
- {pageContents}
- </div>
- </body>
- </html>
-
- get("/") {
- val content =
- <div id="all-sessions">
- <div id="interactive-sessions"></div>
- <div id="batches"></div>
- </div>
-
- createPage("Sessions", content)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/sessions/Session.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/sessions/Session.scala b/server/src/main/scala/com/cloudera/livy/sessions/Session.scala
deleted file mode 100644
index af9ae73..0000000
--- a/server/src/main/scala/com/cloudera/livy/sessions/Session.scala
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.sessions
-
-import java.io.InputStream
-import java.net.{URI, URISyntaxException}
-import java.security.PrivilegedExceptionAction
-import java.util.UUID
-import java.util.concurrent.TimeUnit
-
-import scala.concurrent.{ExecutionContext, Future}
-
-import org.apache.hadoop.fs.{FileSystem, Path}
-import org.apache.hadoop.fs.permission.FsPermission
-import org.apache.hadoop.security.UserGroupInformation
-
-import com.cloudera.livy.{LivyConf, Logging, Utils}
-import com.cloudera.livy.utils.AppInfo
-
-object Session {
- trait RecoveryMetadata { val id: Int }
-
- lazy val configBlackList: Set[String] = {
- val url = getClass.getResource("/spark-blacklist.conf")
- if (url != null) Utils.loadProperties(url).keySet else Set()
- }
-
- /**
- * Validates and prepares a user-provided configuration for submission.
- *
- * - Verifies that no blacklisted configurations are provided.
- * - Merges file lists in the configuration with the explicit lists provided in the request
- * - Resolve file URIs to make sure they reference the default FS
- * - Verify that file URIs don't reference non-whitelisted local resources
- */
- def prepareConf(
- conf: Map[String, String],
- jars: Seq[String],
- files: Seq[String],
- archives: Seq[String],
- pyFiles: Seq[String],
- livyConf: LivyConf): Map[String, String] = {
- if (conf == null) {
- return Map()
- }
-
- val errors = conf.keySet.filter(configBlackList.contains)
- if (errors.nonEmpty) {
- throw new IllegalArgumentException(
- "Blacklisted configuration values in session config: " + errors.mkString(", "))
- }
-
- val confLists: Map[String, Seq[String]] = livyConf.sparkFileLists
- .map { key => (key -> Nil) }.toMap
-
- val userLists = confLists ++ Map(
- LivyConf.SPARK_JARS -> jars,
- LivyConf.SPARK_FILES -> files,
- LivyConf.SPARK_ARCHIVES -> archives,
- LivyConf.SPARK_PY_FILES -> pyFiles)
-
- val merged = userLists.flatMap { case (key, list) =>
- val confList = conf.get(key)
- .map { list =>
- resolveURIs(list.split("[, ]+").toSeq, livyConf)
- }
- .getOrElse(Nil)
- val userList = resolveURIs(list, livyConf)
- if (confList.nonEmpty || userList.nonEmpty) {
- Some(key -> (userList ++ confList).mkString(","))
- } else {
- None
- }
- }
-
- val masterConfList = Map(LivyConf.SPARK_MASTER -> livyConf.sparkMaster()) ++
- livyConf.sparkDeployMode().map(LivyConf.SPARK_DEPLOY_MODE -> _).toMap
-
- conf ++ masterConfList ++ merged
- }
-
- /**
- * Prepends the value of the "fs.defaultFS" configuration to any URIs that do not have a
- * scheme. URIs are required to at least be absolute paths.
- *
- * @throws IllegalArgumentException If an invalid URI is found in the given list.
- */
- def resolveURIs(uris: Seq[String], livyConf: LivyConf): Seq[String] = {
- val defaultFS = livyConf.hadoopConf.get("fs.defaultFS").stripSuffix("/")
- uris.filter(_.nonEmpty).map { _uri =>
- val uri = try {
- new URI(_uri)
- } catch {
- case e: URISyntaxException => throw new IllegalArgumentException(e)
- }
- resolveURI(uri, livyConf).toString()
- }
- }
-
- def resolveURI(uri: URI, livyConf: LivyConf): URI = {
- val defaultFS = livyConf.hadoopConf.get("fs.defaultFS").stripSuffix("/")
- val resolved =
- if (uri.getScheme() == null) {
- require(uri.getPath().startsWith("/"), s"Path '${uri.getPath()}' is not absolute.")
- new URI(defaultFS + uri.getPath())
- } else {
- uri
- }
-
- if (resolved.getScheme() == "file") {
- // Make sure the location is whitelisted before allowing local files to be added.
- require(livyConf.localFsWhitelist.find(resolved.getPath().startsWith).isDefined,
- s"Local path ${uri.getPath()} cannot be added to user sessions.")
- }
-
- resolved
- }
-}
-
-abstract class Session(val id: Int, val owner: String, val livyConf: LivyConf)
- extends Logging {
-
- import Session._
-
- protected implicit val executionContext = ExecutionContext.global
-
- protected var _appId: Option[String] = None
-
- private var _lastActivity = System.nanoTime()
-
- // Directory where the session's staging files are created. The directory is only accessible
- // to the session's effective user.
- private var stagingDir: Path = null
-
- def appId: Option[String] = _appId
-
- var appInfo: AppInfo = AppInfo()
-
- def lastActivity: Long = state match {
- case SessionState.Error(time) => time
- case SessionState.Dead(time) => time
- case SessionState.Success(time) => time
- case _ => _lastActivity
- }
-
- def logLines(): IndexedSeq[String]
-
- def recordActivity(): Unit = {
- _lastActivity = System.nanoTime()
- }
-
- def recoveryMetadata: RecoveryMetadata
-
- def state: SessionState
-
- def stop(): Future[Unit] = Future {
- try {
- info(s"Stopping $this...")
- stopSession()
- info(s"Stopped $this.")
- } catch {
- case e: Exception =>
- warn(s"Error stopping session $id.", e)
- }
-
- try {
- if (stagingDir != null) {
- debug(s"Deleting session $id staging directory $stagingDir")
- doAsOwner {
- val fs = FileSystem.newInstance(livyConf.hadoopConf)
- try {
- fs.delete(stagingDir, true)
- } finally {
- fs.close()
- }
- }
- }
- } catch {
- case e: Exception =>
- warn(s"Error cleaning up session $id staging dir.", e)
- }
- }
-
-
- override def toString(): String = s"${this.getClass.getSimpleName} $id"
-
- protected def stopSession(): Unit
-
- protected val proxyUser: Option[String]
-
- protected def doAsOwner[T](fn: => T): T = {
- val user = proxyUser.getOrElse(owner)
- if (user != null) {
- val ugi = if (UserGroupInformation.isSecurityEnabled) {
- if (livyConf.getBoolean(LivyConf.IMPERSONATION_ENABLED)) {
- UserGroupInformation.createProxyUser(user, UserGroupInformation.getCurrentUser())
- } else {
- UserGroupInformation.getCurrentUser()
- }
- } else {
- UserGroupInformation.createRemoteUser(user)
- }
- ugi.doAs(new PrivilegedExceptionAction[T] {
- override def run(): T = fn
- })
- } else {
- fn
- }
- }
-
- protected def copyResourceToHDFS(dataStream: InputStream, name: String): URI = doAsOwner {
- val fs = FileSystem.newInstance(livyConf.hadoopConf)
-
- try {
- val filePath = new Path(getStagingDir(fs), name)
- debug(s"Uploading user file to $filePath")
-
- val outFile = fs.create(filePath, true)
- val buffer = new Array[Byte](512 * 1024)
- var read = -1
- try {
- while ({read = dataStream.read(buffer); read != -1}) {
- outFile.write(buffer, 0, read)
- }
- } finally {
- outFile.close()
- }
- filePath.toUri
- } finally {
- fs.close()
- }
- }
-
- private def getStagingDir(fs: FileSystem): Path = synchronized {
- if (stagingDir == null) {
- val stagingRoot = Option(livyConf.get(LivyConf.SESSION_STAGING_DIR)).getOrElse {
- new Path(fs.getHomeDirectory(), ".livy-sessions").toString()
- }
-
- val sessionDir = new Path(stagingRoot, UUID.randomUUID().toString())
- fs.mkdirs(sessionDir)
- fs.setPermission(sessionDir, new FsPermission("700"))
- stagingDir = sessionDir
- debug(s"Session $id staging directory is $stagingDir")
- }
- stagingDir
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/sessions/SessionManager.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/sessions/SessionManager.scala b/server/src/main/scala/com/cloudera/livy/sessions/SessionManager.scala
deleted file mode 100644
index 93f162e..0000000
--- a/server/src/main/scala/com/cloudera/livy/sessions/SessionManager.scala
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.sessions
-
-import java.util.concurrent.TimeUnit
-import java.util.concurrent.atomic.AtomicInteger
-
-import scala.collection.mutable
-import scala.concurrent.{Await, ExecutionContext, Future}
-import scala.concurrent.duration.Duration
-import scala.reflect.ClassTag
-import scala.util.control.NonFatal
-
-import com.cloudera.livy.{LivyConf, Logging}
-import com.cloudera.livy.server.batch.{BatchRecoveryMetadata, BatchSession}
-import com.cloudera.livy.server.interactive.{InteractiveRecoveryMetadata, InteractiveSession, SessionHeartbeatWatchdog}
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.Session.RecoveryMetadata
-
-object SessionManager {
- val SESSION_RECOVERY_MODE_OFF = "off"
- val SESSION_RECOVERY_MODE_RECOVERY = "recovery"
-}
-
-class BatchSessionManager(
- livyConf: LivyConf,
- sessionStore: SessionStore,
- mockSessions: Option[Seq[BatchSession]] = None)
- extends SessionManager[BatchSession, BatchRecoveryMetadata] (
- livyConf, BatchSession.recover(_, livyConf, sessionStore), sessionStore, "batch", mockSessions)
-
-class InteractiveSessionManager(
- livyConf: LivyConf,
- sessionStore: SessionStore,
- mockSessions: Option[Seq[InteractiveSession]] = None)
- extends SessionManager[InteractiveSession, InteractiveRecoveryMetadata] (
- livyConf,
- InteractiveSession.recover(_, livyConf, sessionStore),
- sessionStore,
- "interactive",
- mockSessions)
- with SessionHeartbeatWatchdog[InteractiveSession, InteractiveRecoveryMetadata]
- {
- start()
- }
-
-class SessionManager[S <: Session, R <: RecoveryMetadata : ClassTag](
- protected val livyConf: LivyConf,
- sessionRecovery: R => S,
- sessionStore: SessionStore,
- sessionType: String,
- mockSessions: Option[Seq[S]] = None)
- extends Logging {
-
- import SessionManager._
-
- protected implicit def executor: ExecutionContext = ExecutionContext.global
-
- protected[this] final val idCounter = new AtomicInteger(0)
- protected[this] final val sessions = mutable.LinkedHashMap[Int, S]()
-
- private[this] final val sessionTimeoutCheck = livyConf.getBoolean(LivyConf.SESSION_TIMEOUT_CHECK)
- private[this] final val sessionTimeout =
- TimeUnit.MILLISECONDS.toNanos(livyConf.getTimeAsMs(LivyConf.SESSION_TIMEOUT))
- private[this] final val sessionStateRetainedInSec =
- TimeUnit.MILLISECONDS.toNanos(livyConf.getTimeAsMs(LivyConf.SESSION_STATE_RETAIN_TIME))
-
- mockSessions.getOrElse(recover()).foreach(register)
- new GarbageCollector().start()
-
- def nextId(): Int = synchronized {
- val id = idCounter.getAndIncrement()
- sessionStore.saveNextSessionId(sessionType, idCounter.get())
- id
- }
-
- def register(session: S): S = {
- info(s"Registering new session ${session.id}")
- synchronized {
- sessions.put(session.id, session)
- }
- session
- }
-
- def get(id: Int): Option[S] = sessions.get(id)
-
- def size(): Int = sessions.size
-
- def all(): Iterable[S] = sessions.values
-
- def delete(id: Int): Option[Future[Unit]] = {
- get(id).map(delete)
- }
-
- def delete(session: S): Future[Unit] = {
- session.stop().map { case _ =>
- try {
- sessionStore.remove(sessionType, session.id)
- synchronized {
- sessions.remove(session.id)
- }
- } catch {
- case NonFatal(e) =>
- error("Exception was thrown during stop session:", e)
- throw e
- }
- }
- }
-
- def shutdown(): Unit = {
- val recoveryEnabled = livyConf.get(LivyConf.RECOVERY_MODE) != SESSION_RECOVERY_MODE_OFF
- if (!recoveryEnabled) {
- sessions.values.map(_.stop).foreach { future =>
- Await.ready(future, Duration.Inf)
- }
- }
- }
-
- def collectGarbage(): Future[Iterable[Unit]] = {
- def expired(session: Session): Boolean = {
- session.state match {
- case s: FinishedSessionState =>
- val currentTime = System.nanoTime()
- currentTime - s.time > sessionStateRetainedInSec
- case _ =>
- if (!sessionTimeoutCheck) {
- false
- } else if (session.isInstanceOf[BatchSession]) {
- false
- } else {
- val currentTime = System.nanoTime()
- currentTime - session.lastActivity > sessionTimeout
- }
- }
- }
-
- Future.sequence(all().filter(expired).map(delete))
- }
-
- private def recover(): Seq[S] = {
- // Recover next session id from state store and create SessionManager.
- idCounter.set(sessionStore.getNextSessionId(sessionType))
-
- // Retrieve session recovery metadata from state store.
- val sessionMetadata = sessionStore.getAllSessions[R](sessionType)
-
- // Recover session from session recovery metadata.
- val recoveredSessions = sessionMetadata.flatMap(_.toOption).map(sessionRecovery)
-
- info(s"Recovered ${recoveredSessions.length} $sessionType sessions." +
- s" Next session id: $idCounter")
-
- // Print recovery error.
- val recoveryFailure = sessionMetadata.filter(_.isFailure).map(_.failed.get)
- recoveryFailure.foreach(ex => error(ex.getMessage, ex.getCause))
-
- recoveredSessions
- }
-
- private class GarbageCollector extends Thread("session gc thread") {
-
- setDaemon(true)
-
- override def run(): Unit = {
- while (true) {
- collectGarbage()
- Thread.sleep(60 * 1000)
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/utils/Clock.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/utils/Clock.scala b/server/src/main/scala/com/cloudera/livy/utils/Clock.scala
deleted file mode 100644
index 76ad9d4..0000000
--- a/server/src/main/scala/com/cloudera/livy/utils/Clock.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.utils
-
-/**
- * A lot of Livy code relies on time related functions like Thread.sleep.
- * To timing effects from unit test, this class is created to mock out time.
- *
- * Code in Livy should not call Thread.sleep() directly. It should call this class instead.
- */
-object Clock {
- private var _sleep: Long => Unit = Thread.sleep
-
- def withSleepMethod(mockSleep: Long => Unit)(f: => Unit): Unit = {
- try {
- _sleep = mockSleep
- f
- } finally {
- _sleep = Thread.sleep
- }
- }
-
- def sleep: Long => Unit = _sleep
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/utils/LineBufferedProcess.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/utils/LineBufferedProcess.scala b/server/src/main/scala/com/cloudera/livy/utils/LineBufferedProcess.scala
deleted file mode 100644
index a07ab4b..0000000
--- a/server/src/main/scala/com/cloudera/livy/utils/LineBufferedProcess.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.util
-
-import com.cloudera.livy.{Logging, Utils}
-
-class LineBufferedProcess(process: Process) extends Logging {
-
- private[this] val _inputStream = new LineBufferedStream(process.getInputStream)
- private[this] val _errorStream = new LineBufferedStream(process.getErrorStream)
-
- def inputLines: IndexedSeq[String] = _inputStream.lines
- def errorLines: IndexedSeq[String] = _errorStream.lines
-
- def inputIterator: Iterator[String] = _inputStream.iterator
- def errorIterator: Iterator[String] = _errorStream.iterator
-
- def destroy(): Unit = {
- process.destroy()
- }
-
- /** Returns if the process is still actively running. */
- def isAlive: Boolean = Utils.isProcessAlive(process)
-
- def exitValue(): Int = {
- process.exitValue()
- }
-
- def waitFor(): Int = {
- val returnCode = process.waitFor()
- _inputStream.waitUntilClose()
- _errorStream.waitUntilClose()
- returnCode
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/scala/com/cloudera/livy/utils/LineBufferedStream.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/utils/LineBufferedStream.scala b/server/src/main/scala/com/cloudera/livy/utils/LineBufferedStream.scala
deleted file mode 100644
index 5f79ca1..0000000
--- a/server/src/main/scala/com/cloudera/livy/utils/LineBufferedStream.scala
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.util
-
-import java.io.InputStream
-import java.util.concurrent.locks.ReentrantLock
-
-import scala.io.Source
-
-import com.cloudera.livy.Logging
-
-class LineBufferedStream(inputStream: InputStream) extends Logging {
-
- private[this] var _lines: IndexedSeq[String] = IndexedSeq()
-
- private[this] val _lock = new ReentrantLock()
- private[this] val _condition = _lock.newCondition()
- private[this] var _finished = false
-
- private val thread = new Thread {
- override def run() = {
- val lines = Source.fromInputStream(inputStream).getLines()
- for (line <- lines) {
- _lock.lock()
- try {
- _lines = _lines :+ line
- _condition.signalAll()
- } finally {
- _lock.unlock()
- }
- }
-
- _lines.map { line => info("stdout: ", line) }
- _lock.lock()
- try {
- _finished = true
- _condition.signalAll()
- } finally {
- _lock.unlock()
- }
- }
- }
- thread.setDaemon(true)
- thread.start()
-
- def lines: IndexedSeq[String] = _lines
-
- def iterator: Iterator[String] = {
- new LinesIterator
- }
-
- def waitUntilClose(): Unit = thread.join()
-
- private class LinesIterator extends Iterator[String] {
- private[this] var index = 0
-
- override def hasNext: Boolean = {
- if (index < _lines.length) {
- true
- } else {
- // Otherwise we might still have more data.
- _lock.lock()
- try {
- if (_finished) {
- false
- } else {
- _condition.await()
- index < _lines.length
- }
- } finally {
- _lock.unlock()
- }
- }
- }
-
- override def next(): String = {
- val line = _lines(index)
- index += 1
- line
- }
- }
-}
[18/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/bootstrap.min.css
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/bootstrap.min.css b/server/src/main/resources/com/cloudera/livy/server/ui/static/bootstrap.min.css
deleted file mode 100755
index 72cc3ab..0000000
--- a/server/src/main/resources/com/cloudera/livy/server/ui/static/bootstrap.min.css
+++ /dev/null
@@ -1,14 +0,0 @@
-/*!
- * Bootstrap v3.3.7 (http://getbootstrap.com)
- * Copyright 2011-2017 Twitter, Inc.
- * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
- */
-
-/*!
- * Generated using the Bootstrap Customizer (http://getbootstrap.com/customize/?id=794c6d582814e1084501e746ef56e328)
- * Config saved to config.json and https://gist.github.com/794c6d582814e1084501e746ef56e328
- *//*!
- * Bootstrap v3.3.7 (http://getbootstrap.com)
- * Copyright 2011-2016 Twitter, Inc.
- * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
- *//*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,menu,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background-color:transparent}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:bold}dfn{font-style:italic}h1{font-size:2em;margin:0.67em 0}mark{background:#ff0;color:#000}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{border:0}svg:not(:root){overflow:hidden}figure{margin:1em 40px}hr{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;height:0}pre{overflow:auto}code,kbd,pre,samp{font-family:monospace, monospa
ce;font-size:1em}button,input,optgroup,select,textarea{color:inherit;font:inherit;margin:0}button{overflow:visible}button,select{text-transform:none}button,html input[type="button"],input[type="reset"],input[type="submit"]{-webkit-appearance:button;cursor:pointer}button[disabled],html input[disabled]{cursor:default}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}input{line-height:normal}input[type="checkbox"],input[type="radio"]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;padding:0}input[type="number"]::-webkit-inner-spin-button,input[type="number"]::-webkit-outer-spin-button{height:auto}input[type="search"]{-webkit-appearance:textfield;-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box}input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-decoration{-webkit-appearance:none}fieldset{border:1px solid #c0c0c0;margin:0 2px;padding:0.35em 0.625em 0.75em}legend{border:0
;padding:0}textarea{overflow:auto}optgroup{font-weight:bold}table{border-collapse:collapse;border-spacing:0}td,th{padding:0}/*! Source: https://github.com/h5bp/html5-boilerplate/blob/master/src/css/main.css */@media print{*,*:before,*:after{background:transparent !important;color:#000 !important;-webkit-box-shadow:none !important;box-shadow:none !important;text-shadow:none !important}a,a:visited{text-decoration:underline}a[href]:after{content:" (" attr(href) ")"}abbr[title]:after{content:" (" attr(title) ")"}a[href^="#"]:after,a[href^="javascript:"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}p,h2,h3{orphans:3;widows:3}h2,h3{page-break-after:avoid}.navbar{display:none}.btn>.caret,.dropup>.btn>.caret{border-top-color:#000 !important}.label{border:1px solid #000}.table{border-collapse:collapse !important}.table td,.table th{background-color:#fff !important}.t
able-bordered th,.table-bordered td{border:1px solid #ddd !important}}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}*:before,*:after{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}html{font-size:10px;-webkit-tap-highlight-color:rgba(0,0,0,0)}body{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:1.42857143;color:#333;background-color:#fff}input,button,select,textarea{font-family:inherit;font-size:inherit;line-height:inherit}a{color:#337ab7;text-decoration:none}a:hover,a:focus{color:#23527c;text-decoration:underline}a:focus{outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}figure{margin:0}img{vertical-align:middle}.img-responsive,.thumbnail>img,.thumbnail a>img,.carousel-inner>.item>img,.carousel-inner>.item>a>img{display:block;max-width:100%;height:auto}.img-rounded{border-radius:6px}.img-thumbnail{padding:4px;line-height:1.42857143;background-color:#fff;border:1px solid #
ddd;border-radius:4px;-webkit-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out;display:inline-block;max-width:100%;height:auto}.img-circle{border-radius:50%}hr{margin-top:20px;margin-bottom:20px;border:0;border-top:1px solid #eee}.sr-only{position:absolute;width:1px;height:1px;margin:-1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}[role="button"]{cursor:pointer}h1,h2,h3,h4,h5,h6,.h1,.h2,.h3,.h4,.h5,.h6{font-family:inherit;font-weight:500;line-height:1.1;color:inherit}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small,.h1 small,.h2 small,.h3 small,.h4 small,.h5 small,.h6 small,h1 .small,h2 .small,h3 .small,h4 .small,h5 .small,h6 .small,.h1 .small,.h2 .small,.h3 .small,.h4 .small,.h5 .small,.h6 .small{font-weight:normal;line-height:1;color:#777}h1,.h1,h2,.h2,h3,.h3{margin-top:20px;margin-bottom:10p
x}h1 small,.h1 small,h2 small,.h2 small,h3 small,.h3 small,h1 .small,.h1 .small,h2 .small,.h2 .small,h3 .small,.h3 .small{font-size:65%}h4,.h4,h5,.h5,h6,.h6{margin-top:10px;margin-bottom:10px}h4 small,.h4 small,h5 small,.h5 small,h6 small,.h6 small,h4 .small,.h4 .small,h5 .small,.h5 .small,h6 .small,.h6 .small{font-size:75%}h1,.h1{font-size:36px}h2,.h2{font-size:30px}h3,.h3{font-size:24px}h4,.h4{font-size:18px}h5,.h5{font-size:14px}h6,.h6{font-size:12px}p{margin:0 0 10px}.lead{margin-bottom:20px;font-size:16px;font-weight:300;line-height:1.4}@media (min-width:768px){.lead{font-size:21px}}small,.small{font-size:85%}mark,.mark{background-color:#fcf8e3;padding:.2em}.text-left{text-align:left}.text-right{text-align:right}.text-center{text-align:center}.text-justify{text-align:justify}.text-nowrap{white-space:nowrap}.text-lowercase{text-transform:lowercase}.text-uppercase{text-transform:uppercase}.text-capitalize{text-transform:capitalize}.text-muted{color:#777}.text-primary{color:#337ab
7}a.text-primary:hover,a.text-primary:focus{color:#286090}.text-success{color:#3c763d}a.text-success:hover,a.text-success:focus{color:#2b542c}.text-info{color:#31708f}a.text-info:hover,a.text-info:focus{color:#245269}.text-warning{color:#8a6d3b}a.text-warning:hover,a.text-warning:focus{color:#66512c}.text-danger{color:#a94442}a.text-danger:hover,a.text-danger:focus{color:#843534}.bg-primary{color:#fff;background-color:#337ab7}a.bg-primary:hover,a.bg-primary:focus{background-color:#286090}.bg-success{background-color:#dff0d8}a.bg-success:hover,a.bg-success:focus{background-color:#c1e2b3}.bg-info{background-color:#d9edf7}a.bg-info:hover,a.bg-info:focus{background-color:#afd9ee}.bg-warning{background-color:#fcf8e3}a.bg-warning:hover,a.bg-warning:focus{background-color:#f7ecb5}.bg-danger{background-color:#f2dede}a.bg-danger:hover,a.bg-danger:focus{background-color:#e4b9b9}.page-header{padding-bottom:9px;margin:40px 0 20px;border-bottom:1px solid #eee}ul,ol{margin-top:0;margin-bottom:10p
x}ul ul,ol ul,ul ol,ol ol{margin-bottom:0}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none;margin-left:-5px}.list-inline>li{display:inline-block;padding-left:5px;padding-right:5px}dl{margin-top:0;margin-bottom:20px}dt,dd{line-height:1.42857143}dt{font-weight:bold}dd{margin-left:0}@media (min-width:768px){.dl-horizontal dt{float:left;width:160px;clear:left;text-align:right;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:180px}}abbr[title],abbr[data-original-title]{cursor:help;border-bottom:1px dotted #777}.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:10px 20px;margin:0 0 20px;font-size:17.5px;border-left:5px solid #eee}blockquote p:last-child,blockquote ul:last-child,blockquote ol:last-child{margin-bottom:0}blockquote footer,blockquote small,blockquote .small{display:block;font-size:80%;line-height:1.42857143;color:#777}blockquote footer:before,blockquote small:before,blockquote .
small:before{content:'\2014 \00A0'}.blockquote-reverse,blockquote.pull-right{padding-right:15px;padding-left:0;border-right:5px solid #eee;border-left:0;text-align:right}.blockquote-reverse footer:before,blockquote.pull-right footer:before,.blockquote-reverse small:before,blockquote.pull-right small:before,.blockquote-reverse .small:before,blockquote.pull-right .small:before{content:''}.blockquote-reverse footer:after,blockquote.pull-right footer:after,.blockquote-reverse small:after,blockquote.pull-right small:after,.blockquote-reverse .small:after,blockquote.pull-right .small:after{content:'\00A0 \2014'}address{margin-bottom:20px;font-style:normal;line-height:1.42857143}code,kbd,pre,samp{font-family:Menlo,Monaco,Consolas,"Courier New",monospace}code{padding:2px 4px;font-size:90%;color:#c7254e;background-color:#f9f2f4;border-radius:4px}kbd{padding:2px 4px;font-size:90%;color:#fff;background-color:#333;border-radius:3px;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.25);box-shadow:i
nset 0 -1px 0 rgba(0,0,0,0.25)}kbd kbd{padding:0;font-size:100%;font-weight:bold;-webkit-box-shadow:none;box-shadow:none}pre{display:block;padding:9.5px;margin:0 0 10px;font-size:13px;line-height:1.42857143;word-break:break-all;word-wrap:break-word;color:#333;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px}pre code{padding:0;font-size:inherit;color:inherit;white-space:pre-wrap;background-color:transparent;border-radius:0}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{margin-right:auto;margin-left:auto;padding-left:15px;padding-right:15px}@media (min-width:768px){.container{width:750px}}@media (min-width:992px){.container{width:970px}}@media (min-width:1200px){.container{width:1170px}}.container-fluid{margin-right:auto;margin-left:auto;padding-left:15px;padding-right:15px}.row{margin-left:-15px;margin-right:-15px}.col-xs-1, .col-sm-1, .col-md-1, .col-lg-1, .col-xs-2, .col-sm-2, .col-md-2, .col-lg-2, .col-xs-3, .col-sm-3, .col-md-3, .col-lg-3, .col-xs-4
, .col-sm-4, .col-md-4, .col-lg-4, .col-xs-5, .col-sm-5, .col-md-5, .col-lg-5, .col-xs-6, .col-sm-6, .col-md-6, .col-lg-6, .col-xs-7, .col-sm-7, .col-md-7, .col-lg-7, .col-xs-8, .col-sm-8, .col-md-8, .col-lg-8, .col-xs-9, .col-sm-9, .col-md-9, .col-lg-9, .col-xs-10, .col-sm-10, .col-md-10, .col-lg-10, .col-xs-11, .col-sm-11, .col-md-11, .col-lg-11, .col-xs-12, .col-sm-12, .col-md-12, .col-lg-12{position:relative;min-height:1px;padding-left:15px;padding-right:15px}.col-xs-1, .col-xs-2, .col-xs-3, .col-xs-4, .col-xs-5, .col-xs-6, .col-xs-7, .col-xs-8, .col-xs-9, .col-xs-10, .col-xs-11, .col-xs-12{float:left}.col-xs-12{width:100%}.col-xs-11{width:91.66666667%}.col-xs-10{width:83.33333333%}.col-xs-9{width:75%}.col-xs-8{width:66.66666667%}.col-xs-7{width:58.33333333%}.col-xs-6{width:50%}.col-xs-5{width:41.66666667%}.col-xs-4{width:33.33333333%}.col-xs-3{width:25%}.col-xs-2{width:16.66666667%}.col-xs-1{width:8.33333333%}.col-xs-pull-12{right:100%}.col-xs-pull-11{right:91.66666667%}.col-xs
-pull-10{right:83.33333333%}.col-xs-pull-9{right:75%}.col-xs-pull-8{right:66.66666667%}.col-xs-pull-7{right:58.33333333%}.col-xs-pull-6{right:50%}.col-xs-pull-5{right:41.66666667%}.col-xs-pull-4{right:33.33333333%}.col-xs-pull-3{right:25%}.col-xs-pull-2{right:16.66666667%}.col-xs-pull-1{right:8.33333333%}.col-xs-pull-0{right:auto}.col-xs-push-12{left:100%}.col-xs-push-11{left:91.66666667%}.col-xs-push-10{left:83.33333333%}.col-xs-push-9{left:75%}.col-xs-push-8{left:66.66666667%}.col-xs-push-7{left:58.33333333%}.col-xs-push-6{left:50%}.col-xs-push-5{left:41.66666667%}.col-xs-push-4{left:33.33333333%}.col-xs-push-3{left:25%}.col-xs-push-2{left:16.66666667%}.col-xs-push-1{left:8.33333333%}.col-xs-push-0{left:auto}.col-xs-offset-12{margin-left:100%}.col-xs-offset-11{margin-left:91.66666667%}.col-xs-offset-10{margin-left:83.33333333%}.col-xs-offset-9{margin-left:75%}.col-xs-offset-8{margin-left:66.66666667%}.col-xs-offset-7{margin-left:58.33333333%}.col-xs-offset-6{margin-left:50%}.col-x
s-offset-5{margin-left:41.66666667%}.col-xs-offset-4{margin-left:33.33333333%}.col-xs-offset-3{margin-left:25%}.col-xs-offset-2{margin-left:16.66666667%}.col-xs-offset-1{margin-left:8.33333333%}.col-xs-offset-0{margin-left:0}@media (min-width:768px){.col-sm-1, .col-sm-2, .col-sm-3, .col-sm-4, .col-sm-5, .col-sm-6, .col-sm-7, .col-sm-8, .col-sm-9, .col-sm-10, .col-sm-11, .col-sm-12{float:left}.col-sm-12{width:100%}.col-sm-11{width:91.66666667%}.col-sm-10{width:83.33333333%}.col-sm-9{width:75%}.col-sm-8{width:66.66666667%}.col-sm-7{width:58.33333333%}.col-sm-6{width:50%}.col-sm-5{width:41.66666667%}.col-sm-4{width:33.33333333%}.col-sm-3{width:25%}.col-sm-2{width:16.66666667%}.col-sm-1{width:8.33333333%}.col-sm-pull-12{right:100%}.col-sm-pull-11{right:91.66666667%}.col-sm-pull-10{right:83.33333333%}.col-sm-pull-9{right:75%}.col-sm-pull-8{right:66.66666667%}.col-sm-pull-7{right:58.33333333%}.col-sm-pull-6{right:50%}.col-sm-pull-5{right:41.66666667%}.col-sm-pull-4{right:33.33333333%}.col
-sm-pull-3{right:25%}.col-sm-pull-2{right:16.66666667%}.col-sm-pull-1{right:8.33333333%}.col-sm-pull-0{right:auto}.col-sm-push-12{left:100%}.col-sm-push-11{left:91.66666667%}.col-sm-push-10{left:83.33333333%}.col-sm-push-9{left:75%}.col-sm-push-8{left:66.66666667%}.col-sm-push-7{left:58.33333333%}.col-sm-push-6{left:50%}.col-sm-push-5{left:41.66666667%}.col-sm-push-4{left:33.33333333%}.col-sm-push-3{left:25%}.col-sm-push-2{left:16.66666667%}.col-sm-push-1{left:8.33333333%}.col-sm-push-0{left:auto}.col-sm-offset-12{margin-left:100%}.col-sm-offset-11{margin-left:91.66666667%}.col-sm-offset-10{margin-left:83.33333333%}.col-sm-offset-9{margin-left:75%}.col-sm-offset-8{margin-left:66.66666667%}.col-sm-offset-7{margin-left:58.33333333%}.col-sm-offset-6{margin-left:50%}.col-sm-offset-5{margin-left:41.66666667%}.col-sm-offset-4{margin-left:33.33333333%}.col-sm-offset-3{margin-left:25%}.col-sm-offset-2{margin-left:16.66666667%}.col-sm-offset-1{margin-left:8.33333333%}.col-sm-offset-0{margin-
left:0}}@media (min-width:992px){.col-md-1, .col-md-2, .col-md-3, .col-md-4, .col-md-5, .col-md-6, .col-md-7, .col-md-8, .col-md-9, .col-md-10, .col-md-11, .col-md-12{float:left}.col-md-12{width:100%}.col-md-11{width:91.66666667%}.col-md-10{width:83.33333333%}.col-md-9{width:75%}.col-md-8{width:66.66666667%}.col-md-7{width:58.33333333%}.col-md-6{width:50%}.col-md-5{width:41.66666667%}.col-md-4{width:33.33333333%}.col-md-3{width:25%}.col-md-2{width:16.66666667%}.col-md-1{width:8.33333333%}.col-md-pull-12{right:100%}.col-md-pull-11{right:91.66666667%}.col-md-pull-10{right:83.33333333%}.col-md-pull-9{right:75%}.col-md-pull-8{right:66.66666667%}.col-md-pull-7{right:58.33333333%}.col-md-pull-6{right:50%}.col-md-pull-5{right:41.66666667%}.col-md-pull-4{right:33.33333333%}.col-md-pull-3{right:25%}.col-md-pull-2{right:16.66666667%}.col-md-pull-1{right:8.33333333%}.col-md-pull-0{right:auto}.col-md-push-12{left:100%}.col-md-push-11{left:91.66666667%}.col-md-push-10{left:83.33333333%}.col-md-p
ush-9{left:75%}.col-md-push-8{left:66.66666667%}.col-md-push-7{left:58.33333333%}.col-md-push-6{left:50%}.col-md-push-5{left:41.66666667%}.col-md-push-4{left:33.33333333%}.col-md-push-3{left:25%}.col-md-push-2{left:16.66666667%}.col-md-push-1{left:8.33333333%}.col-md-push-0{left:auto}.col-md-offset-12{margin-left:100%}.col-md-offset-11{margin-left:91.66666667%}.col-md-offset-10{margin-left:83.33333333%}.col-md-offset-9{margin-left:75%}.col-md-offset-8{margin-left:66.66666667%}.col-md-offset-7{margin-left:58.33333333%}.col-md-offset-6{margin-left:50%}.col-md-offset-5{margin-left:41.66666667%}.col-md-offset-4{margin-left:33.33333333%}.col-md-offset-3{margin-left:25%}.col-md-offset-2{margin-left:16.66666667%}.col-md-offset-1{margin-left:8.33333333%}.col-md-offset-0{margin-left:0}}@media (min-width:1200px){.col-lg-1, .col-lg-2, .col-lg-3, .col-lg-4, .col-lg-5, .col-lg-6, .col-lg-7, .col-lg-8, .col-lg-9, .col-lg-10, .col-lg-11, .col-lg-12{float:left}.col-lg-12{width:100%}.col-lg-11{width
:91.66666667%}.col-lg-10{width:83.33333333%}.col-lg-9{width:75%}.col-lg-8{width:66.66666667%}.col-lg-7{width:58.33333333%}.col-lg-6{width:50%}.col-lg-5{width:41.66666667%}.col-lg-4{width:33.33333333%}.col-lg-3{width:25%}.col-lg-2{width:16.66666667%}.col-lg-1{width:8.33333333%}.col-lg-pull-12{right:100%}.col-lg-pull-11{right:91.66666667%}.col-lg-pull-10{right:83.33333333%}.col-lg-pull-9{right:75%}.col-lg-pull-8{right:66.66666667%}.col-lg-pull-7{right:58.33333333%}.col-lg-pull-6{right:50%}.col-lg-pull-5{right:41.66666667%}.col-lg-pull-4{right:33.33333333%}.col-lg-pull-3{right:25%}.col-lg-pull-2{right:16.66666667%}.col-lg-pull-1{right:8.33333333%}.col-lg-pull-0{right:auto}.col-lg-push-12{left:100%}.col-lg-push-11{left:91.66666667%}.col-lg-push-10{left:83.33333333%}.col-lg-push-9{left:75%}.col-lg-push-8{left:66.66666667%}.col-lg-push-7{left:58.33333333%}.col-lg-push-6{left:50%}.col-lg-push-5{left:41.66666667%}.col-lg-push-4{left:33.33333333%}.col-lg-push-3{left:25%}.col-lg-push-2{left:1
6.66666667%}.col-lg-push-1{left:8.33333333%}.col-lg-push-0{left:auto}.col-lg-offset-12{margin-left:100%}.col-lg-offset-11{margin-left:91.66666667%}.col-lg-offset-10{margin-left:83.33333333%}.col-lg-offset-9{margin-left:75%}.col-lg-offset-8{margin-left:66.66666667%}.col-lg-offset-7{margin-left:58.33333333%}.col-lg-offset-6{margin-left:50%}.col-lg-offset-5{margin-left:41.66666667%}.col-lg-offset-4{margin-left:33.33333333%}.col-lg-offset-3{margin-left:25%}.col-lg-offset-2{margin-left:16.66666667%}.col-lg-offset-1{margin-left:8.33333333%}.col-lg-offset-0{margin-left:0}}table{background-color:transparent}caption{padding-top:8px;padding-bottom:8px;color:#777;text-align:left}th{text-align:left}.table{width:100%;max-width:100%;margin-bottom:20px}.table>thead>tr>th,.table>tbody>tr>th,.table>tfoot>tr>th,.table>thead>tr>td,.table>tbody>tr>td,.table>tfoot>tr>td{padding:8px;line-height:1.42857143;vertical-align:top;border-top:1px solid #ddd}.table>thead>tr>th{vertical-align:bottom;border-bottom:
2px solid #ddd}.table>caption+thead>tr:first-child>th,.table>colgroup+thead>tr:first-child>th,.table>thead:first-child>tr:first-child>th,.table>caption+thead>tr:first-child>td,.table>colgroup+thead>tr:first-child>td,.table>thead:first-child>tr:first-child>td{border-top:0}.table>tbody+tbody{border-top:2px solid #ddd}.table .table{background-color:#fff}.table-condensed>thead>tr>th,.table-condensed>tbody>tr>th,.table-condensed>tfoot>tr>th,.table-condensed>thead>tr>td,.table-condensed>tbody>tr>td,.table-condensed>tfoot>tr>td{padding:5px}.table-bordered{border:1px solid #ddd}.table-bordered>thead>tr>th,.table-bordered>tbody>tr>th,.table-bordered>tfoot>tr>th,.table-bordered>thead>tr>td,.table-bordered>tbody>tr>td,.table-bordered>tfoot>tr>td{border:1px solid #ddd}.table-bordered>thead>tr>th,.table-bordered>thead>tr>td{border-bottom-width:2px}.table-striped>tbody>tr:nth-of-type(odd){background-color:#f9f9f9}.table-hover>tbody>tr:hover{background-color:#f5f5f5}table col[class*="col-"]{positi
on:static;float:none;display:table-column}table td[class*="col-"],table th[class*="col-"]{position:static;float:none;display:table-cell}.table>thead>tr>td.active,.table>tbody>tr>td.active,.table>tfoot>tr>td.active,.table>thead>tr>th.active,.table>tbody>tr>th.active,.table>tfoot>tr>th.active,.table>thead>tr.active>td,.table>tbody>tr.active>td,.table>tfoot>tr.active>td,.table>thead>tr.active>th,.table>tbody>tr.active>th,.table>tfoot>tr.active>th{background-color:#f5f5f5}.table-hover>tbody>tr>td.active:hover,.table-hover>tbody>tr>th.active:hover,.table-hover>tbody>tr.active:hover>td,.table-hover>tbody>tr:hover>.active,.table-hover>tbody>tr.active:hover>th{background-color:#e8e8e8}.table>thead>tr>td.success,.table>tbody>tr>td.success,.table>tfoot>tr>td.success,.table>thead>tr>th.success,.table>tbody>tr>th.success,.table>tfoot>tr>th.success,.table>thead>tr.success>td,.table>tbody>tr.success>td,.table>tfoot>tr.success>td,.table>thead>tr.success>th,.table>tbody>tr.success>th,.table>tfoot>t
r.success>th{background-color:#dff0d8}.table-hover>tbody>tr>td.success:hover,.table-hover>tbody>tr>th.success:hover,.table-hover>tbody>tr.success:hover>td,.table-hover>tbody>tr:hover>.success,.table-hover>tbody>tr.success:hover>th{background-color:#d0e9c6}.table>thead>tr>td.info,.table>tbody>tr>td.info,.table>tfoot>tr>td.info,.table>thead>tr>th.info,.table>tbody>tr>th.info,.table>tfoot>tr>th.info,.table>thead>tr.info>td,.table>tbody>tr.info>td,.table>tfoot>tr.info>td,.table>thead>tr.info>th,.table>tbody>tr.info>th,.table>tfoot>tr.info>th{background-color:#d9edf7}.table-hover>tbody>tr>td.info:hover,.table-hover>tbody>tr>th.info:hover,.table-hover>tbody>tr.info:hover>td,.table-hover>tbody>tr:hover>.info,.table-hover>tbody>tr.info:hover>th{background-color:#c4e3f3}.table>thead>tr>td.warning,.table>tbody>tr>td.warning,.table>tfoot>tr>td.warning,.table>thead>tr>th.warning,.table>tbody>tr>th.warning,.table>tfoot>tr>th.warning,.table>thead>tr.warning>td,.table>tbody>tr.warning>td,.table>tf
oot>tr.warning>td,.table>thead>tr.warning>th,.table>tbody>tr.warning>th,.table>tfoot>tr.warning>th{background-color:#fcf8e3}.table-hover>tbody>tr>td.warning:hover,.table-hover>tbody>tr>th.warning:hover,.table-hover>tbody>tr.warning:hover>td,.table-hover>tbody>tr:hover>.warning,.table-hover>tbody>tr.warning:hover>th{background-color:#faf2cc}.table>thead>tr>td.danger,.table>tbody>tr>td.danger,.table>tfoot>tr>td.danger,.table>thead>tr>th.danger,.table>tbody>tr>th.danger,.table>tfoot>tr>th.danger,.table>thead>tr.danger>td,.table>tbody>tr.danger>td,.table>tfoot>tr.danger>td,.table>thead>tr.danger>th,.table>tbody>tr.danger>th,.table>tfoot>tr.danger>th{background-color:#f2dede}.table-hover>tbody>tr>td.danger:hover,.table-hover>tbody>tr>th.danger:hover,.table-hover>tbody>tr.danger:hover>td,.table-hover>tbody>tr:hover>.danger,.table-hover>tbody>tr.danger:hover>th{background-color:#ebcccc}.table-responsive{overflow-x:auto;min-height:0.01%}@media screen and (max-width:767px){.table-responsive{
width:100%;margin-bottom:15px;overflow-y:hidden;-ms-overflow-style:-ms-autohiding-scrollbar;border:1px solid #ddd}.table-responsive>.table{margin-bottom:0}.table-responsive>.table>thead>tr>th,.table-responsive>.table>tbody>tr>th,.table-responsive>.table>tfoot>tr>th,.table-responsive>.table>thead>tr>td,.table-responsive>.table>tbody>tr>td,.table-responsive>.table>tfoot>tr>td{white-space:nowrap}.table-responsive>.table-bordered{border:0}.table-responsive>.table-bordered>thead>tr>th:first-child,.table-responsive>.table-bordered>tbody>tr>th:first-child,.table-responsive>.table-bordered>tfoot>tr>th:first-child,.table-responsive>.table-bordered>thead>tr>td:first-child,.table-responsive>.table-bordered>tbody>tr>td:first-child,.table-responsive>.table-bordered>tfoot>tr>td:first-child{border-left:0}.table-responsive>.table-bordered>thead>tr>th:last-child,.table-responsive>.table-bordered>tbody>tr>th:last-child,.table-responsive>.table-bordered>tfoot>tr>th:last-child,.table-responsive>.table-
bordered>thead>tr>td:last-child,.table-responsive>.table-bordered>tbody>tr>td:last-child,.table-responsive>.table-bordered>tfoot>tr>td:last-child{border-right:0}.table-responsive>.table-bordered>tbody>tr:last-child>th,.table-responsive>.table-bordered>tfoot>tr:last-child>th,.table-responsive>.table-bordered>tbody>tr:last-child>td,.table-responsive>.table-bordered>tfoot>tr:last-child>td{border-bottom:0}}fieldset{padding:0;margin:0;border:0;min-width:0}legend{display:block;width:100%;padding:0;margin-bottom:20px;font-size:21px;line-height:inherit;color:#333;border:0;border-bottom:1px solid #e5e5e5}label{display:inline-block;max-width:100%;margin-bottom:5px;font-weight:bold}input[type="search"]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type="radio"],input[type="checkbox"]{margin:4px 0 0;margin-top:1px \9;line-height:normal}input[type="file"]{display:block}input[type="range"]{display:block;width:100%}select[multiple],select[size]{height:auto}in
put[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus{outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}output{display:block;padding-top:7px;font-size:14px;line-height:1.42857143;color:#555}.form-control{display:block;width:100%;height:34px;padding:6px 12px;font-size:14px;line-height:1.42857143;color:#555;background-color:#fff;background-image:none;border:1px solid #ccc;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-webkit-transition:border-color ease-in-out .15s, -webkit-box-shadow ease-in-out .15s;-o-transition:border-color ease-in-out .15s, box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s, box-shadow ease-in-out .15s}.form-control:focus{border-color:#66afe9;outline:0;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6);box-shadow:inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6)}.form-control::-moz-placehol
der{color:#999;opacity:1}.form-control:-ms-input-placeholder{color:#999}.form-control::-webkit-input-placeholder{color:#999}.form-control::-ms-expand{border:0;background-color:transparent}.form-control[disabled],.form-control[readonly],fieldset[disabled] .form-control{background-color:#eee;opacity:1}.form-control[disabled],fieldset[disabled] .form-control{cursor:not-allowed}textarea.form-control{height:auto}input[type="search"]{-webkit-appearance:none}@media screen and (-webkit-min-device-pixel-ratio:0){input[type="date"].form-control,input[type="time"].form-control,input[type="datetime-local"].form-control,input[type="month"].form-control{line-height:34px}input[type="date"].input-sm,input[type="time"].input-sm,input[type="datetime-local"].input-sm,input[type="month"].input-sm,.input-group-sm input[type="date"],.input-group-sm input[type="time"],.input-group-sm input[type="datetime-local"],.input-group-sm input[type="month"]{line-height:30px}input[type="date"].input-lg,input[type="t
ime"].input-lg,input[type="datetime-local"].input-lg,input[type="month"].input-lg,.input-group-lg input[type="date"],.input-group-lg input[type="time"],.input-group-lg input[type="datetime-local"],.input-group-lg input[type="month"]{line-height:46px}}.form-group{margin-bottom:15px}.radio,.checkbox{position:relative;display:block;margin-top:10px;margin-bottom:10px}.radio label,.checkbox label{min-height:20px;padding-left:20px;margin-bottom:0;font-weight:normal;cursor:pointer}.radio input[type="radio"],.radio-inline input[type="radio"],.checkbox input[type="checkbox"],.checkbox-inline input[type="checkbox"]{position:absolute;margin-left:-20px;margin-top:4px \9}.radio+.radio,.checkbox+.checkbox{margin-top:-5px}.radio-inline,.checkbox-inline{position:relative;display:inline-block;padding-left:20px;margin-bottom:0;vertical-align:middle;font-weight:normal;cursor:pointer}.radio-inline+.radio-inline,.checkbox-inline+.checkbox-inline{margin-top:0;margin-left:10px}input[type="radio"][disabled
],input[type="checkbox"][disabled],input[type="radio"].disabled,input[type="checkbox"].disabled,fieldset[disabled] input[type="radio"],fieldset[disabled] input[type="checkbox"]{cursor:not-allowed}.radio-inline.disabled,.checkbox-inline.disabled,fieldset[disabled] .radio-inline,fieldset[disabled] .checkbox-inline{cursor:not-allowed}.radio.disabled label,.checkbox.disabled label,fieldset[disabled] .radio label,fieldset[disabled] .checkbox label{cursor:not-allowed}.form-control-static{padding-top:7px;padding-bottom:7px;margin-bottom:0;min-height:34px}.form-control-static.input-lg,.form-control-static.input-sm{padding-left:0;padding-right:0}.input-sm{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}select.input-sm{height:30px;line-height:30px}textarea.input-sm,select[multiple].input-sm{height:auto}.form-group-sm .form-control{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.form-group-sm select.form-control{height:30px;line-heig
ht:30px}.form-group-sm textarea.form-control,.form-group-sm select[multiple].form-control{height:auto}.form-group-sm .form-control-static{height:30px;min-height:32px;padding:6px 10px;font-size:12px;line-height:1.5}.input-lg{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}select.input-lg{height:46px;line-height:46px}textarea.input-lg,select[multiple].input-lg{height:auto}.form-group-lg .form-control{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.form-group-lg select.form-control{height:46px;line-height:46px}.form-group-lg textarea.form-control,.form-group-lg select[multiple].form-control{height:auto}.form-group-lg .form-control-static{height:46px;min-height:38px;padding:11px 16px;font-size:18px;line-height:1.3333333}.has-feedback{position:relative}.has-feedback .form-control{padding-right:42.5px}.form-control-feedback{position:absolute;top:0;right:0;z-index:2;display:block;width:34px;height:34px;line-height:3
4px;text-align:center;pointer-events:none}.input-lg+.form-control-feedback,.input-group-lg+.form-control-feedback,.form-group-lg .form-control+.form-control-feedback{width:46px;height:46px;line-height:46px}.input-sm+.form-control-feedback,.input-group-sm+.form-control-feedback,.form-group-sm .form-control+.form-control-feedback{width:30px;height:30px;line-height:30px}.has-success .help-block,.has-success .control-label,.has-success .radio,.has-success .checkbox,.has-success .radio-inline,.has-success .checkbox-inline,.has-success.radio label,.has-success.checkbox label,.has-success.radio-inline label,.has-success.checkbox-inline label{color:#3c763d}.has-success .form-control{border-color:#3c763d;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.has-success .form-control:focus{border-color:#2b542c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #67b168;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #67b168}.has-succe
ss .input-group-addon{color:#3c763d;border-color:#3c763d;background-color:#dff0d8}.has-success .form-control-feedback{color:#3c763d}.has-warning .help-block,.has-warning .control-label,.has-warning .radio,.has-warning .checkbox,.has-warning .radio-inline,.has-warning .checkbox-inline,.has-warning.radio label,.has-warning.checkbox label,.has-warning.radio-inline label,.has-warning.checkbox-inline label{color:#8a6d3b}.has-warning .form-control{border-color:#8a6d3b;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.has-warning .form-control:focus{border-color:#66512c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #c0a16b;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #c0a16b}.has-warning .input-group-addon{color:#8a6d3b;border-color:#8a6d3b;background-color:#fcf8e3}.has-warning .form-control-feedback{color:#8a6d3b}.has-error .help-block,.has-error .control-label,.has-error .radio,.has-error .checkbox,.has-error .radio-
inline,.has-error .checkbox-inline,.has-error.radio label,.has-error.checkbox label,.has-error.radio-inline label,.has-error.checkbox-inline label{color:#a94442}.has-error .form-control{border-color:#a94442;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.has-error .form-control:focus{border-color:#843534;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #ce8483;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #ce8483}.has-error .input-group-addon{color:#a94442;border-color:#a94442;background-color:#f2dede}.has-error .form-control-feedback{color:#a94442}.has-feedback label~.form-control-feedback{top:25px}.has-feedback label.sr-only~.form-control-feedback{top:0}.help-block{display:block;margin-top:5px;margin-bottom:10px;color:#737373}@media (min-width:768px){.form-inline .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle
}.form-inline .form-control-static{display:inline-block}.form-inline .input-group{display:inline-table;vertical-align:middle}.form-inline .input-group .input-group-addon,.form-inline .input-group .input-group-btn,.form-inline .input-group .form-control{width:auto}.form-inline .input-group>.form-control{width:100%}.form-inline .control-label{margin-bottom:0;vertical-align:middle}.form-inline .radio,.form-inline .checkbox{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle}.form-inline .radio label,.form-inline .checkbox label{padding-left:0}.form-inline .radio input[type="radio"],.form-inline .checkbox input[type="checkbox"]{position:relative;margin-left:0}.form-inline .has-feedback .form-control-feedback{top:0}}.form-horizontal .radio,.form-horizontal .checkbox,.form-horizontal .radio-inline,.form-horizontal .checkbox-inline{margin-top:0;margin-bottom:0;padding-top:7px}.form-horizontal .radio,.form-horizontal .checkbox{min-height:27px}.form-horizontal .form-group
{margin-left:-15px;margin-right:-15px}@media (min-width:768px){.form-horizontal .control-label{text-align:right;margin-bottom:0;padding-top:7px}}.form-horizontal .has-feedback .form-control-feedback{right:15px}@media (min-width:768px){.form-horizontal .form-group-lg .control-label{padding-top:11px;font-size:18px}}@media (min-width:768px){.form-horizontal .form-group-sm .control-label{padding-top:6px;font-size:12px}}.btn{display:inline-block;margin-bottom:0;font-weight:normal;text-align:center;vertical-align:middle;-ms-touch-action:manipulation;touch-action:manipulation;cursor:pointer;background-image:none;border:1px solid transparent;white-space:nowrap;padding:6px 12px;font-size:14px;line-height:1.42857143;border-radius:4px;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.btn:focus,.btn:active:focus,.btn.active:focus,.btn.focus,.btn:active.focus,.btn.active.focus{outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn:hover,.btn:focus,
.btn.focus{color:#333;text-decoration:none}.btn:active,.btn.active{outline:0;background-image:none;-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,0.125);box-shadow:inset 0 3px 5px rgba(0,0,0,0.125)}.btn.disabled,.btn[disabled],fieldset[disabled] .btn{cursor:not-allowed;opacity:.65;filter:alpha(opacity=65);-webkit-box-shadow:none;box-shadow:none}a.btn.disabled,fieldset[disabled] a.btn{pointer-events:none}.btn-default{color:#333;background-color:#fff;border-color:#ccc}.btn-default:focus,.btn-default.focus{color:#333;background-color:#e6e6e6;border-color:#8c8c8c}.btn-default:hover{color:#333;background-color:#e6e6e6;border-color:#adadad}.btn-default:active,.btn-default.active,.open>.dropdown-toggle.btn-default{color:#333;background-color:#e6e6e6;border-color:#adadad}.btn-default:active:hover,.btn-default.active:hover,.open>.dropdown-toggle.btn-default:hover,.btn-default:active:focus,.btn-default.active:focus,.open>.dropdown-toggle.btn-default:focus,.btn-default:active.focus,.btn-default
.active.focus,.open>.dropdown-toggle.btn-default.focus{color:#333;background-color:#d4d4d4;border-color:#8c8c8c}.btn-default:active,.btn-default.active,.open>.dropdown-toggle.btn-default{background-image:none}.btn-default.disabled:hover,.btn-default[disabled]:hover,fieldset[disabled] .btn-default:hover,.btn-default.disabled:focus,.btn-default[disabled]:focus,fieldset[disabled] .btn-default:focus,.btn-default.disabled.focus,.btn-default[disabled].focus,fieldset[disabled] .btn-default.focus{background-color:#fff;border-color:#ccc}.btn-default .badge{color:#fff;background-color:#333}.btn-primary{color:#fff;background-color:#337ab7;border-color:#2e6da4}.btn-primary:focus,.btn-primary.focus{color:#fff;background-color:#286090;border-color:#122b40}.btn-primary:hover{color:#fff;background-color:#286090;border-color:#204d74}.btn-primary:active,.btn-primary.active,.open>.dropdown-toggle.btn-primary{color:#fff;background-color:#286090;border-color:#204d74}.btn-primary:active:hover,.btn-primar
y.active:hover,.open>.dropdown-toggle.btn-primary:hover,.btn-primary:active:focus,.btn-primary.active:focus,.open>.dropdown-toggle.btn-primary:focus,.btn-primary:active.focus,.btn-primary.active.focus,.open>.dropdown-toggle.btn-primary.focus{color:#fff;background-color:#204d74;border-color:#122b40}.btn-primary:active,.btn-primary.active,.open>.dropdown-toggle.btn-primary{background-image:none}.btn-primary.disabled:hover,.btn-primary[disabled]:hover,fieldset[disabled] .btn-primary:hover,.btn-primary.disabled:focus,.btn-primary[disabled]:focus,fieldset[disabled] .btn-primary:focus,.btn-primary.disabled.focus,.btn-primary[disabled].focus,fieldset[disabled] .btn-primary.focus{background-color:#337ab7;border-color:#2e6da4}.btn-primary .badge{color:#337ab7;background-color:#fff}.btn-success{color:#fff;background-color:#5cb85c;border-color:#4cae4c}.btn-success:focus,.btn-success.focus{color:#fff;background-color:#449d44;border-color:#255625}.btn-success:hover{color:#fff;background-color:#4
49d44;border-color:#398439}.btn-success:active,.btn-success.active,.open>.dropdown-toggle.btn-success{color:#fff;background-color:#449d44;border-color:#398439}.btn-success:active:hover,.btn-success.active:hover,.open>.dropdown-toggle.btn-success:hover,.btn-success:active:focus,.btn-success.active:focus,.open>.dropdown-toggle.btn-success:focus,.btn-success:active.focus,.btn-success.active.focus,.open>.dropdown-toggle.btn-success.focus{color:#fff;background-color:#398439;border-color:#255625}.btn-success:active,.btn-success.active,.open>.dropdown-toggle.btn-success{background-image:none}.btn-success.disabled:hover,.btn-success[disabled]:hover,fieldset[disabled] .btn-success:hover,.btn-success.disabled:focus,.btn-success[disabled]:focus,fieldset[disabled] .btn-success:focus,.btn-success.disabled.focus,.btn-success[disabled].focus,fieldset[disabled] .btn-success.focus{background-color:#5cb85c;border-color:#4cae4c}.btn-success .badge{color:#5cb85c;background-color:#fff}.btn-info{color:#f
ff;background-color:#5bc0de;border-color:#46b8da}.btn-info:focus,.btn-info.focus{color:#fff;background-color:#31b0d5;border-color:#1b6d85}.btn-info:hover{color:#fff;background-color:#31b0d5;border-color:#269abc}.btn-info:active,.btn-info.active,.open>.dropdown-toggle.btn-info{color:#fff;background-color:#31b0d5;border-color:#269abc}.btn-info:active:hover,.btn-info.active:hover,.open>.dropdown-toggle.btn-info:hover,.btn-info:active:focus,.btn-info.active:focus,.open>.dropdown-toggle.btn-info:focus,.btn-info:active.focus,.btn-info.active.focus,.open>.dropdown-toggle.btn-info.focus{color:#fff;background-color:#269abc;border-color:#1b6d85}.btn-info:active,.btn-info.active,.open>.dropdown-toggle.btn-info{background-image:none}.btn-info.disabled:hover,.btn-info[disabled]:hover,fieldset[disabled] .btn-info:hover,.btn-info.disabled:focus,.btn-info[disabled]:focus,fieldset[disabled] .btn-info:focus,.btn-info.disabled.focus,.btn-info[disabled].focus,fieldset[disabled] .btn-info.focus{backgrou
nd-color:#5bc0de;border-color:#46b8da}.btn-info .badge{color:#5bc0de;background-color:#fff}.btn-warning{color:#fff;background-color:#f0ad4e;border-color:#eea236}.btn-warning:focus,.btn-warning.focus{color:#fff;background-color:#ec971f;border-color:#985f0d}.btn-warning:hover{color:#fff;background-color:#ec971f;border-color:#d58512}.btn-warning:active,.btn-warning.active,.open>.dropdown-toggle.btn-warning{color:#fff;background-color:#ec971f;border-color:#d58512}.btn-warning:active:hover,.btn-warning.active:hover,.open>.dropdown-toggle.btn-warning:hover,.btn-warning:active:focus,.btn-warning.active:focus,.open>.dropdown-toggle.btn-warning:focus,.btn-warning:active.focus,.btn-warning.active.focus,.open>.dropdown-toggle.btn-warning.focus{color:#fff;background-color:#d58512;border-color:#985f0d}.btn-warning:active,.btn-warning.active,.open>.dropdown-toggle.btn-warning{background-image:none}.btn-warning.disabled:hover,.btn-warning[disabled]:hover,fieldset[disabled] .btn-warning:hover,.btn-
warning.disabled:focus,.btn-warning[disabled]:focus,fieldset[disabled] .btn-warning:focus,.btn-warning.disabled.focus,.btn-warning[disabled].focus,fieldset[disabled] .btn-warning.focus{background-color:#f0ad4e;border-color:#eea236}.btn-warning .badge{color:#f0ad4e;background-color:#fff}.btn-danger{color:#fff;background-color:#d9534f;border-color:#d43f3a}.btn-danger:focus,.btn-danger.focus{color:#fff;background-color:#c9302c;border-color:#761c19}.btn-danger:hover{color:#fff;background-color:#c9302c;border-color:#ac2925}.btn-danger:active,.btn-danger.active,.open>.dropdown-toggle.btn-danger{color:#fff;background-color:#c9302c;border-color:#ac2925}.btn-danger:active:hover,.btn-danger.active:hover,.open>.dropdown-toggle.btn-danger:hover,.btn-danger:active:focus,.btn-danger.active:focus,.open>.dropdown-toggle.btn-danger:focus,.btn-danger:active.focus,.btn-danger.active.focus,.open>.dropdown-toggle.btn-danger.focus{color:#fff;background-color:#ac2925;border-color:#761c19}.btn-danger:activ
e,.btn-danger.active,.open>.dropdown-toggle.btn-danger{background-image:none}.btn-danger.disabled:hover,.btn-danger[disabled]:hover,fieldset[disabled] .btn-danger:hover,.btn-danger.disabled:focus,.btn-danger[disabled]:focus,fieldset[disabled] .btn-danger:focus,.btn-danger.disabled.focus,.btn-danger[disabled].focus,fieldset[disabled] .btn-danger.focus{background-color:#d9534f;border-color:#d43f3a}.btn-danger .badge{color:#d9534f;background-color:#fff}.btn-link{color:#337ab7;font-weight:normal;border-radius:0}.btn-link,.btn-link:active,.btn-link.active,.btn-link[disabled],fieldset[disabled] .btn-link{background-color:transparent;-webkit-box-shadow:none;box-shadow:none}.btn-link,.btn-link:hover,.btn-link:focus,.btn-link:active{border-color:transparent}.btn-link:hover,.btn-link:focus{color:#23527c;text-decoration:underline;background-color:transparent}.btn-link[disabled]:hover,fieldset[disabled] .btn-link:hover,.btn-link[disabled]:focus,fieldset[disabled] .btn-link:focus{color:#777;text
-decoration:none}.btn-lg,.btn-group-lg>.btn{padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.btn-sm,.btn-group-sm>.btn{padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.btn-xs,.btn-group-xs>.btn{padding:1px 5px;font-size:12px;line-height:1.5;border-radius:3px}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:5px}input[type="submit"].btn-block,input[type="reset"].btn-block,input[type="button"].btn-block{width:100%}.fade{opacity:0;-webkit-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1}.collapse{display:none}.collapse.in{display:block}tr.collapse.in{display:table-row}tbody.collapse.in{display:table-row-group}.collapsing{position:relative;height:0;overflow:hidden;-webkit-transition-property:height, visibility;-o-transition-property:height, visibility;transition-property:height, visibility;-webkit-transition-duration:.35s;-o-transition-duration:.35s;transition-
duration:.35s;-webkit-transition-timing-function:ease;-o-transition-timing-function:ease;transition-timing-function:ease}.caret{display:inline-block;width:0;height:0;margin-left:2px;vertical-align:middle;border-top:4px dashed;border-top:4px solid \9;border-right:4px solid transparent;border-left:4px solid transparent}.dropup,.dropdown{position:relative}.dropdown-toggle:focus{outline:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;list-style:none;font-size:14px;text-align:left;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.15);border-radius:4px;-webkit-box-shadow:0 6px 12px rgba(0,0,0,0.175);box-shadow:0 6px 12px rgba(0,0,0,0.175);-webkit-background-clip:padding-box;background-clip:padding-box}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5}.dropdown-menu>li>a{display:block;padding:3px 20px
;clear:both;font-weight:normal;line-height:1.42857143;color:#333;white-space:nowrap}.dropdown-menu>li>a:hover,.dropdown-menu>li>a:focus{text-decoration:none;color:#262626;background-color:#f5f5f5}.dropdown-menu>.active>a,.dropdown-menu>.active>a:hover,.dropdown-menu>.active>a:focus{color:#fff;text-decoration:none;outline:0;background-color:#337ab7}.dropdown-menu>.disabled>a,.dropdown-menu>.disabled>a:hover,.dropdown-menu>.disabled>a:focus{color:#777}.dropdown-menu>.disabled>a:hover,.dropdown-menu>.disabled>a:focus{text-decoration:none;background-color:transparent;background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);cursor:not-allowed}.open>.dropdown-menu{display:block}.open>a{outline:0}.dropdown-menu-right{left:auto;right:0}.dropdown-menu-left{left:0;right:auto}.dropdown-header{display:block;padding:3px 20px;font-size:12px;line-height:1.42857143;color:#777;white-space:nowrap}.dropdown-backdrop{position:fixed;left:0;right:0;bottom:0;top:0;z-index:9
90}.pull-right>.dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0;border-bottom:4px dashed;border-bottom:4px solid \9;content:""}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:2px}@media (min-width:768px){.navbar-right .dropdown-menu{left:auto;right:0}.navbar-right .dropdown-menu-left{left:0;right:auto}}.btn-group,.btn-group-vertical{position:relative;display:inline-block;vertical-align:middle}.btn-group>.btn,.btn-group-vertical>.btn{position:relative;float:left}.btn-group>.btn:hover,.btn-group-vertical>.btn:hover,.btn-group>.btn:focus,.btn-group-vertical>.btn:focus,.btn-group>.btn:active,.btn-group-vertical>.btn:active,.btn-group>.btn.active,.btn-group-vertical>.btn.active{z-index:2}.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group{margin-left:-1px}.btn-toolbar{margin-left:-5px}.btn-toolbar .btn,.btn-toolbar .btn-group,.b
tn-toolbar .input-group{float:left}.btn-toolbar>.btn,.btn-toolbar>.btn-group,.btn-toolbar>.input-group{margin-left:5px}.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle){border-radius:0}.btn-group>.btn:first-child{margin-left:0}.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-top-right-radius:0}.btn-group>.btn:last-child:not(:first-child),.btn-group>.dropdown-toggle:not(:first-child){border-bottom-left-radius:0;border-top-left-radius:0}.btn-group>.btn-group{float:left}.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-top-right-radius:0}.btn-group>.btn-group:last-child:not(:first-child)>.btn:first-child{border-bottom-left-radius:0;border-top-left-radius:0}.btn-group .dropdown-toggle:active,.btn-group.open .dropdow
n-toggle{outline:0}.btn-group>.btn+.dropdown-toggle{padding-left:8px;padding-right:8px}.btn-group>.btn-lg+.dropdown-toggle{padding-left:12px;padding-right:12px}.btn-group.open .dropdown-toggle{-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,0.125);box-shadow:inset 0 3px 5px rgba(0,0,0,0.125)}.btn-group.open .dropdown-toggle.btn-link{-webkit-box-shadow:none;box-shadow:none}.btn .caret{margin-left:0}.btn-lg .caret{border-width:5px 5px 0;border-bottom-width:0}.dropup .btn-lg .caret{border-width:0 5px 5px}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group,.btn-group-vertical>.btn-group>.btn{display:block;float:none;width:100%;max-width:100%}.btn-group-vertical>.btn-group>.btn{float:none}.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0}.btn-group-vertical>.btn:not(:first-child):not(:last-child){border-radius:0}.btn-group-vertical>.btn:first-child:not(:last-child
){border-top-right-radius:4px;border-top-left-radius:4px;border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:last-child:not(:first-child){border-top-right-radius:0;border-top-left-radius:0;border-bottom-right-radius:4px;border-bottom-left-radius:4px}.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-right-radius:0;border-top-left-radius:0}.btn-group-justified{display:table;width:100%;table-layout:fixed;border-collapse:separate}.btn-group-justified>.btn,.btn-group-justified>.btn-group{float:none;display:table-cell;width:1%}.btn-group-justified>.btn-group .btn{width:100%}.btn-group-justified>.btn-group .dropdown-menu{left:auto}
[data-toggle="buttons"]>.btn input[type="radio"],[data-toggle="buttons"]>.btn-group>.btn input[type="radio"],[data-toggle="buttons"]>.btn input[type="checkbox"],[data-toggle="buttons"]>.btn-group>.btn input[type="checkbox"]{position:absolute;clip:rect(0, 0, 0, 0);pointer-events:none}.input-group{position:relative;display:table;border-collapse:separate}.input-group[class*="col-"]{float:none;padding-left:0;padding-right:0}.input-group .form-control{position:relative;z-index:2;float:left;width:100%;margin-bottom:0}.input-group .form-control:focus{z-index:3}.input-group-lg>.form-control,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.btn{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}select.input-group-lg>.form-control,select.input-group-lg>.input-group-addon,select.input-group-lg>.input-group-btn>.btn{height:46px;line-height:46px}textarea.input-group-lg>.form-control,textarea.input-group-lg>.input-group-addon,textarea.input-gro
up-lg>.input-group-btn>.btn,select[multiple].input-group-lg>.form-control,select[multiple].input-group-lg>.input-group-addon,select[multiple].input-group-lg>.input-group-btn>.btn{height:auto}.input-group-sm>.form-control,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.btn{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}select.input-group-sm>.form-control,select.input-group-sm>.input-group-addon,select.input-group-sm>.input-group-btn>.btn{height:30px;line-height:30px}textarea.input-group-sm>.form-control,textarea.input-group-sm>.input-group-addon,textarea.input-group-sm>.input-group-btn>.btn,select[multiple].input-group-sm>.form-control,select[multiple].input-group-sm>.input-group-addon,select[multiple].input-group-sm>.input-group-btn>.btn{height:auto}.input-group-addon,.input-group-btn,.input-group .form-control{display:table-cell}.input-group-addon:not(:first-child):not(:last-child),.input-group-btn:not(:first-child):not(:last-chil
d),.input-group .form-control:not(:first-child):not(:last-child){border-radius:0}.input-group-addon,.input-group-btn{width:1%;white-space:nowrap;vertical-align:middle}.input-group-addon{padding:6px 12px;font-size:14px;font-weight:normal;line-height:1;color:#555;text-align:center;background-color:#eee;border:1px solid #ccc;border-radius:4px}.input-group-addon.input-sm{padding:5px 10px;font-size:12px;border-radius:3px}.input-group-addon.input-lg{padding:10px 16px;font-size:18px;border-radius:6px}.input-group-addon input[type="radio"],.input-group-addon input[type="checkbox"]{margin-top:0}.input-group .form-control:first-child,.input-group-addon:first-child,.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group>.btn,.input-group-btn:first-child>.dropdown-toggle,.input-group-btn:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group-btn:last-child>.btn-group:not(:last-child)>.btn{border-bottom-right-radius:0;border-top-right-radius:0}.input-group-addon:fi
rst-child{border-right:0}.input-group .form-control:last-child,.input-group-addon:last-child,.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group>.btn,.input-group-btn:last-child>.dropdown-toggle,.input-group-btn:first-child>.btn:not(:first-child),.input-group-btn:first-child>.btn-group:not(:first-child)>.btn{border-bottom-left-radius:0;border-top-left-radius:0}.input-group-addon:last-child{border-left:0}.input-group-btn{position:relative;font-size:0;white-space:nowrap}.input-group-btn>.btn{position:relative}.input-group-btn>.btn+.btn{margin-left:-1px}.input-group-btn>.btn:hover,.input-group-btn>.btn:focus,.input-group-btn>.btn:active{z-index:2}.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group{margin-right:-1px}.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group{z-index:2;margin-left:-1px}.nav{margin-bottom:0;padding-left:0;list-style:none}.nav>li{position:relative;display:block}.nav>li>a{position:relative;display:block;
padding:10px 15px}.nav>li>a:hover,.nav>li>a:focus{text-decoration:none;background-color:#eee}.nav>li.disabled>a{color:#777}.nav>li.disabled>a:hover,.nav>li.disabled>a:focus{color:#777;text-decoration:none;background-color:transparent;cursor:not-allowed}.nav .open>a,.nav .open>a:hover,.nav .open>a:focus{background-color:#eee;border-color:#337ab7}.nav .nav-divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5}.nav>li>a>img{max-width:none}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{float:left;margin-bottom:-1px}.nav-tabs>li>a{margin-right:2px;line-height:1.42857143;border:1px solid transparent;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>li.active>a,.nav-tabs>li.active>a:hover,.nav-tabs>li.active>a:focus{color:#555;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent;cursor:default}.nav-tabs.nav-justified{width:100%;border-bottom:0}.nav-tabs.nav-justified>li{float:none}.nav-tabs.nav-justified>li>a
{text-align:center;margin-bottom:5px}.nav-tabs.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-tabs.nav-justified>li{display:table-cell;width:1%}.nav-tabs.nav-justified>li>a{margin-bottom:0}}.nav-tabs.nav-justified>li>a{margin-right:0;border-radius:4px}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:hover,.nav-tabs.nav-justified>.active>a:focus{border:1px solid #ddd}@media (min-width:768px){.nav-tabs.nav-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:hover,.nav-tabs.nav-justified>.active>a:focus{border-bottom-color:#fff}}.nav-pills>li{float:left}.nav-pills>li>a{border-radius:4px}.nav-pills>li+li{margin-left:2px}.nav-pills>li.active>a,.nav-pills>li.active>a:hover,.nav-pills>li.active>a:focus{color:#fff;background-color:#337ab7}.nav-stacked>li{float:none}.nav-stacked>li+li{margin-top:2px;margin-left:0}.nav-justified{width:100%}.nav-just
ified>li{float:none}.nav-justified>li>a{text-align:center;margin-bottom:5px}.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-justified>li{display:table-cell;width:1%}.nav-justified>li>a{margin-bottom:0}}.nav-tabs-justified{border-bottom:0}.nav-tabs-justified>li>a{margin-right:0;border-radius:4px}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:hover,.nav-tabs-justified>.active>a:focus{border:1px solid #ddd}@media (min-width:768px){.nav-tabs-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:hover,.nav-tabs-justified>.active>a:focus{border-bottom-color:#fff}}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-right-radius:0;border-top-left-radius:0}.navbar{position:relative;min-height:50px;margin-bottom:20px;border:1px solid transparent}@media (min-width:768px){.navbar{border-radius:4px
}}@media (min-width:768px){.navbar-header{float:left}}.navbar-collapse{overflow-x:visible;padding-right:15px;padding-left:15px;border-top:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1);-webkit-overflow-scrolling:touch}.navbar-collapse.in{overflow-y:auto}@media (min-width:768px){.navbar-collapse{width:auto;border-top:0;-webkit-box-shadow:none;box-shadow:none}.navbar-collapse.collapse{display:block !important;height:auto !important;padding-bottom:0;overflow:visible !important}.navbar-collapse.in{overflow-y:visible}.navbar-fixed-top .navbar-collapse,.navbar-static-top .navbar-collapse,.navbar-fixed-bottom .navbar-collapse{padding-left:0;padding-right:0}}.navbar-fixed-top .navbar-collapse,.navbar-fixed-bottom .navbar-collapse{max-height:340px}@media (max-device-width:480px) and (orientation:landscape){.navbar-fixed-top .navbar-collapse,.navbar-fixed-bottom .navbar-collapse{max-height:200px}}.container>.navbar-h
eader,.container-fluid>.navbar-header,.container>.navbar-collapse,.container-fluid>.navbar-collapse{margin-right:-15px;margin-left:-15px}@media (min-width:768px){.container>.navbar-header,.container-fluid>.navbar-header,.container>.navbar-collapse,.container-fluid>.navbar-collapse{margin-right:0;margin-left:0}}.navbar-static-top{z-index:1000;border-width:0 0 1px}@media (min-width:768px){.navbar-static-top{border-radius:0}}.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;z-index:1030}@media (min-width:768px){.navbar-fixed-top,.navbar-fixed-bottom{border-radius:0}}.navbar-fixed-top{top:0;border-width:0 0 1px}.navbar-fixed-bottom{bottom:0;margin-bottom:0;border-width:1px 0 0}.navbar-brand{float:left;padding:15px 15px;font-size:18px;line-height:20px;height:50px}.navbar-brand:hover,.navbar-brand:focus{text-decoration:none}.navbar-brand>img{display:block}@media (min-width:768px){.navbar>.container .navbar-brand,.navbar>.container-fluid .navbar-brand{margin-left:-15px}}
.navbar-toggle{position:relative;float:right;margin-right:15px;padding:9px 10px;margin-top:8px;margin-bottom:8px;background-color:transparent;background-image:none;border:1px solid transparent;border-radius:4px}.navbar-toggle:focus{outline:0}.navbar-toggle .icon-bar{display:block;width:22px;height:2px;border-radius:1px}.navbar-toggle .icon-bar+.icon-bar{margin-top:4px}@media (min-width:768px){.navbar-toggle{display:none}}.navbar-nav{margin:7.5px -15px}.navbar-nav>li>a{padding-top:10px;padding-bottom:10px;line-height:20px}@media (max-width:767px){.navbar-nav .open .dropdown-menu{position:static;float:none;width:auto;margin-top:0;background-color:transparent;border:0;-webkit-box-shadow:none;box-shadow:none}.navbar-nav .open .dropdown-menu>li>a,.navbar-nav .open .dropdown-menu .dropdown-header{padding:5px 15px 5px 25px}.navbar-nav .open .dropdown-menu>li>a{line-height:20px}.navbar-nav .open .dropdown-menu>li>a:hover,.navbar-nav .open .dropdown-menu>li>a:focus{background-image:none}}@me
dia (min-width:768px){.navbar-nav{float:left;margin:0}.navbar-nav>li{float:left}.navbar-nav>li>a{padding-top:15px;padding-bottom:15px}}.navbar-form{margin-left:-15px;margin-right:-15px;padding:10px 15px;border-top:1px solid transparent;border-bottom:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);margin-top:8px;margin-bottom:8px}@media (min-width:768px){.navbar-form .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.navbar-form .form-control{display:inline-block;width:auto;vertical-align:middle}.navbar-form .form-control-static{display:inline-block}.navbar-form .input-group{display:inline-table;vertical-align:middle}.navbar-form .input-group .input-group-addon,.navbar-form .input-group .input-group-btn,.navbar-form .input-group .form-control{width:auto}.navbar-form .input-group>.form-control{width:100%}.navbar-form .control-labe
l{margin-bottom:0;vertical-align:middle}.navbar-form .radio,.navbar-form .checkbox{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle}.navbar-form .radio label,.navbar-form .checkbox label{padding-left:0}.navbar-form .radio input[type="radio"],.navbar-form .checkbox input[type="checkbox"]{position:relative;margin-left:0}.navbar-form .has-feedback .form-control-feedback{top:0}}@media (max-width:767px){.navbar-form .form-group{margin-bottom:5px}.navbar-form .form-group:last-child{margin-bottom:0}}@media (min-width:768px){.navbar-form{width:auto;border:0;margin-left:0;margin-right:0;padding-top:0;padding-bottom:0;-webkit-box-shadow:none;box-shadow:none}}.navbar-nav>li>.dropdown-menu{margin-top:0;border-top-right-radius:0;border-top-left-radius:0}.navbar-fixed-bottom .navbar-nav>li>.dropdown-menu{margin-bottom:0;border-top-right-radius:4px;border-top-left-radius:4px;border-bottom-right-radius:0;border-bottom-left-radius:0}.navbar-btn{margin-top:8px;margin-bottom:8px
}.navbar-btn.btn-sm{margin-top:10px;margin-bottom:10px}.navbar-btn.btn-xs{margin-top:14px;margin-bottom:14px}.navbar-text{margin-top:15px;margin-bottom:15px}@media (min-width:768px){.navbar-text{float:left;margin-left:15px;margin-right:15px}}@media (min-width:768px){.navbar-left{float:left !important}.navbar-right{float:right !important;margin-right:-15px}.navbar-right~.navbar-right{margin-right:0}}.navbar-default{background-color:#f8f8f8;border-color:#e7e7e7}.navbar-default .navbar-brand{color:#777}.navbar-default .navbar-brand:hover,.navbar-default .navbar-brand:focus{color:#5e5e5e;background-color:transparent}.navbar-default .navbar-text{color:#777}.navbar-default .navbar-nav>li>a{color:#777}.navbar-default .navbar-nav>li>a:hover,.navbar-default .navbar-nav>li>a:focus{color:#333;background-color:transparent}.navbar-default .navbar-nav>.active>a,.navbar-default .navbar-nav>.active>a:hover,.navbar-default .navbar-nav>.active>a:focus{color:#555;background-color:#e7e7e7}.navbar-defau
lt .navbar-nav>.disabled>a,.navbar-default .navbar-nav>.disabled>a:hover,.navbar-default .navbar-nav>.disabled>a:focus{color:#ccc;background-color:transparent}.navbar-default .navbar-toggle{border-color:#ddd}.navbar-default .navbar-toggle:hover,.navbar-default .navbar-toggle:focus{background-color:#ddd}.navbar-default .navbar-toggle .icon-bar{background-color:#888}.navbar-default .navbar-collapse,.navbar-default .navbar-form{border-color:#e7e7e7}.navbar-default .navbar-nav>.open>a,.navbar-default .navbar-nav>.open>a:hover,.navbar-default .navbar-nav>.open>a:focus{background-color:#e7e7e7;color:#555}@media (max-width:767px){.navbar-default .navbar-nav .open .dropdown-menu>li>a{color:#777}.navbar-default .navbar-nav .open .dropdown-menu>li>a:hover,.navbar-default .navbar-nav .open .dropdown-menu>li>a:focus{color:#333;background-color:transparent}.navbar-default .navbar-nav .open .dropdown-menu>.active>a,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:hover,.navbar-default .
navbar-nav .open .dropdown-menu>.active>a:focus{color:#555;background-color:#e7e7e7}.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:hover,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:focus{color:#ccc;background-color:transparent}}.navbar-default .navbar-link{color:#777}.navbar-default .navbar-link:hover{color:#333}.navbar-default .btn-link{color:#777}.navbar-default .btn-link:hover,.navbar-default .btn-link:focus{color:#333}.navbar-default .btn-link[disabled]:hover,fieldset[disabled] .navbar-default .btn-link:hover,.navbar-default .btn-link[disabled]:focus,fieldset[disabled] .navbar-default .btn-link:focus{color:#ccc}.navbar-inverse{background-color:#222;border-color:#080808}.navbar-inverse .navbar-brand{color:#9d9d9d}.navbar-inverse .navbar-brand:hover,.navbar-inverse .navbar-brand:focus{color:#fff;background-color:transparent}.navbar-inverse .navbar-text{color:#9d9d9d}.navbar-inverse .navbar-
nav>li>a{color:#9d9d9d}.navbar-inverse .navbar-nav>li>a:hover,.navbar-inverse .navbar-nav>li>a:focus{color:#fff;background-color:transparent}.navbar-inverse .navbar-nav>.active>a,.navbar-inverse .navbar-nav>.active>a:hover,.navbar-inverse .navbar-nav>.active>a:focus{color:#fff;background-color:#080808}.navbar-inverse .navbar-nav>.disabled>a,.navbar-inverse .navbar-nav>.disabled>a:hover,.navbar-inverse .navbar-nav>.disabled>a:focus{color:#444;background-color:transparent}.navbar-inverse .navbar-toggle{border-color:#333}.navbar-inverse .navbar-toggle:hover,.navbar-inverse .navbar-toggle:focus{background-color:#333}.navbar-inverse .navbar-toggle .icon-bar{background-color:#fff}.navbar-inverse .navbar-collapse,.navbar-inverse .navbar-form{border-color:#101010}.navbar-inverse .navbar-nav>.open>a,.navbar-inverse .navbar-nav>.open>a:hover,.navbar-inverse .navbar-nav>.open>a:focus{background-color:#080808;color:#fff}@media (max-width:767px){.navbar-inverse .navbar-nav .open .dropdown-menu>.
dropdown-header{border-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu .divider{background-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a{color:#9d9d9d}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:hover,.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:focus{color:#fff;background-color:transparent}.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:hover,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:focus{color:#fff;background-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:hover,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:focus{color:#444;background-color:transparent}}.navbar-inverse .navbar-link{color:#9d9d9d}.navbar-inverse .navbar-link:hover{color:#fff}.navbar-inverse .btn-link{color:#9d9d9d}.navbar-inverse .btn-link:hover,.navbar-inverse .btn-link:f
ocus{color:#fff}.navbar-inverse .btn-link[disabled]:hover,fieldset[disabled] .navbar-inverse .btn-link:hover,.navbar-inverse .btn-link[disabled]:focus,fieldset[disabled] .navbar-inverse .btn-link:focus{color:#444}.breadcrumb{padding:8px 15px;margin-bottom:20px;list-style:none;background-color:#f5f5f5;border-radius:4px}.breadcrumb>li{display:inline-block}.breadcrumb>li+li:before{content:"/\00a0";padding:0 5px;color:#ccc}.breadcrumb>.active{color:#777}.pagination{display:inline-block;padding-left:0;margin:20px 0;border-radius:4px}.pagination>li{display:inline}.pagination>li>a,.pagination>li>span{position:relative;float:left;padding:6px 12px;line-height:1.42857143;text-decoration:none;color:#337ab7;background-color:#fff;border:1px solid #ddd;margin-left:-1px}.pagination>li:first-child>a,.pagination>li:first-child>span{margin-left:0;border-bottom-left-radius:4px;border-top-left-radius:4px}.pagination>li:last-child>a,.pagination>li:last-child>span{border-bottom-right-radius:4px;border-to
p-right-radius:4px}.pagination>li>a:hover,.pagination>li>span:hover,.pagination>li>a:focus,.pagination>li>span:focus{z-index:2;color:#23527c;background-color:#eee;border-color:#ddd}.pagination>.active>a,.pagination>.active>span,.pagination>.active>a:hover,.pagination>.active>span:hover,.pagination>.active>a:focus,.pagination>.active>span:focus{z-index:3;color:#fff;background-color:#337ab7;border-color:#337ab7;cursor:default}.pagination>.disabled>span,.pagination>.disabled>span:hover,.pagination>.disabled>span:focus,.pagination>.disabled>a,.pagination>.disabled>a:hover,.pagination>.disabled>a:focus{color:#777;background-color:#fff;border-color:#ddd;cursor:not-allowed}.pagination-lg>li>a,.pagination-lg>li>span{padding:10px 16px;font-size:18px;line-height:1.3333333}.pagination-lg>li:first-child>a,.pagination-lg>li:first-child>span{border-bottom-left-radius:6px;border-top-left-radius:6px}.pagination-lg>li:last-child>a,.pagination-lg>li:last-child>span{border-bottom-right-radius:6px;bord
er-top-right-radius:6px}.pagination-sm>li>a,.pagination-sm>li>span{padding:5px 10px;font-size:12px;line-height:1.5}.pagination-sm>li:first-child>a,.pagination-sm>li:first-child>span{border-bottom-left-radius:3px;border-top-left-radius:3px}.pagination-sm>li:last-child>a,.pagination-sm>li:last-child>span{border-bottom-right-radius:3px;border-top-right-radius:3px}.pager{padding-left:0;margin:20px 0;list-style:none;text-align:center}.pager li{display:inline}.pager li>a,.pager li>span{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;border-radius:15px}.pager li>a:hover,.pager li>a:focus{text-decoration:none;background-color:#eee}.pager .next>a,.pager .next>span{float:right}.pager .previous>a,.pager .previous>span{float:left}.pager .disabled>a,.pager .disabled>a:hover,.pager .disabled>a:focus,.pager .disabled>span{color:#777;background-color:#fff;cursor:not-allowed}.label{display:inline;padding:.2em .6em .3em;font-size:75%;font-weight:bold;line-height:1;co
lor:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25em}a.label:hover,a.label:focus{color:#fff;text-decoration:none;cursor:pointer}.label:empty{display:none}.btn .label{position:relative;top:-1px}.label-default{background-color:#777}.label-default[href]:hover,.label-default[href]:focus{background-color:#5e5e5e}.label-primary{background-color:#337ab7}.label-primary[href]:hover,.label-primary[href]:focus{background-color:#286090}.label-success{background-color:#5cb85c}.label-success[href]:hover,.label-success[href]:focus{background-color:#449d44}.label-info{background-color:#5bc0de}.label-info[href]:hover,.label-info[href]:focus{background-color:#31b0d5}.label-warning{background-color:#f0ad4e}.label-warning[href]:hover,.label-warning[href]:focus{background-color:#ec971f}.label-danger{background-color:#d9534f}.label-danger[href]:hover,.label-danger[href]:focus{background-color:#c9302c}.badge{display:inline-block;min-width:10px;padding:3px 7px;font-size
:12px;font-weight:bold;color:#fff;line-height:1;vertical-align:middle;white-space:nowrap;text-align:center;background-color:#777;border-radius:10px}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.btn-xs .badge,.btn-group-xs>.btn .badge{top:0;padding:1px 5px}a.badge:hover,a.badge:focus{color:#fff;text-decoration:none;cursor:pointer}.list-group-item.active>.badge,.nav-pills>.active>a>.badge{color:#337ab7;background-color:#fff}.list-group-item>.badge{float:right}.list-group-item>.badge+.badge{margin-right:5px}.nav-pills>li>a>.badge{margin-left:3px}.jumbotron{padding-top:30px;padding-bottom:30px;margin-bottom:30px;color:inherit;background-color:#eee}.jumbotron h1,.jumbotron .h1{color:inherit}.jumbotron p{margin-bottom:15px;font-size:21px;font-weight:200}.jumbotron>hr{border-top-color:#d5d5d5}.container .jumbotron,.container-fluid .jumbotron{border-radius:6px;padding-left:15px;padding-right:15px}.jumbotron .container{max-width:100%}@media screen and (min-width:768px){.j
umbotron{padding-top:48px;padding-bottom:48px}.container .jumbotron,.container-fluid .jumbotron{padding-left:60px;padding-right:60px}.jumbotron h1,.jumbotron .h1{font-size:63px}}.thumbnail{display:block;padding:4px;margin-bottom:20px;line-height:1.42857143;background-color:#fff;border:1px solid #ddd;border-radius:4px;-webkit-transition:border .2s ease-in-out;-o-transition:border .2s ease-in-out;transition:border .2s ease-in-out}.thumbnail>img,.thumbnail a>img{margin-left:auto;margin-right:auto}a.thumbnail:hover,a.thumbnail:focus,a.thumbnail.active{border-color:#337ab7}.thumbnail .caption{padding:9px;color:#333}.alert{padding:15px;margin-bottom:20px;border:1px solid transparent;border-radius:4px}.alert h4{margin-top:0;color:inherit}.alert .alert-link{font-weight:bold}.alert>p,.alert>ul{margin-bottom:0}.alert>p+p{margin-top:5px}.alert-dismissable,.alert-dismissible{padding-right:35px}.alert-dismissable .close,.alert-dismissible .close{position:relative;top:-2px;right:-21px;color:inher
it}.alert-success{background-color:#dff0d8;border-color:#d6e9c6;color:#3c763d}.alert-success hr{border-top-color:#c9e2b3}.alert-success .alert-link{color:#2b542c}.alert-info{background-color:#d9edf7;border-color:#bce8f1;color:#31708f}.alert-info hr{border-top-color:#a6e1ec}.alert-info .alert-link{color:#245269}.alert-warning{background-color:#fcf8e3;border-color:#faebcc;color:#8a6d3b}.alert-warning hr{border-top-color:#f7e1b5}.alert-warning .alert-link{color:#66512c}.alert-danger{background-color:#f2dede;border-color:#ebccd1;color:#a94442}.alert-danger hr{border-top-color:#e4b9c0}.alert-danger .alert-link{color:#843534}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{overflow:hidden;height:20px;margin-bottom:20px;background-color:#f5f5f5;b
order-radius:4px;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1)}.progress-bar{float:left;width:0%;height:100%;font-size:12px;line-height:20px;color:#fff;text-align:center;background-color:#337ab7;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress-striped .progress-bar,.progress-bar-striped{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%,
rgba(255,255,255,0.15) 75%, transparent 75%, transparent);-webkit-background-size:40px 40px;background-size:40px 40px}.progress.active .progress-bar,.progress-bar.active{-webkit-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-bar-success{background-color:#5cb85c}.progress-striped .progress-bar-success{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.pro
gress-bar-info{background-color:#5bc0de}.progress-striped .progress-bar-info{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.progress-bar-warning{background-color:#f0ad4e}.progress-striped .progress-bar-warning{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255,255,255,0.
15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.progress-bar-danger{background-color:#d9534f}.progress-striped .progress-bar-danger{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.media{margin-top:15
px}.media:first-child{margin-top:0}.media,.media-body{zoom:1;overflow:hidden}.media-body{width:10000px}.media-object{display:block}.media-object.img-thumbnail{max-width:none}.media-right,.media>.pull-right{padding-left:10px}.media-left,.media>.pull-left{padding-right:10px}.media-left,.media-right,.media-body{display:table-cell;vertical-align:top}.media-middle{vertical-align:middle}.media-bottom{vertical-align:bottom}.media-heading{margin-top:0;margin-bottom:5px}.media-list{padding-left:0;list-style:none}.list-group{margin-bottom:20px;padding-left:0}.list-group-item{position:relative;display:block;padding:10px 15px;margin-bottom:-1px;background-color:#fff;border:1px solid #ddd}.list-group-item:first-child{border-top-right-radius:4px;border-top-left-radius:4px}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:4px;border-bottom-left-radius:4px}a.list-group-item,button.list-group-item{color:#555}a.list-group-item .list-group-item-heading,button.list-group-item .list
-group-item-heading{color:#333}a.list-group-item:hover,button.list-group-item:hover,a.list-group-item:focus,button.list-group-item:focus{text-decoration:none;color:#555;background-color:#f5f5f5}button.list-group-item{width:100%;text-align:left}.list-group-item.disabled,.list-group-item.disabled:hover,.list-group-item.disabled:focus{background-color:#eee;color:#777;cursor:not-allowed}.list-group-item.disabled .list-group-item-heading,.list-group-item.disabled:hover .list-group-item-heading,.list-group-item.disabled:focus .list-group-item-heading{color:inherit}.list-group-item.disabled .list-group-item-text,.list-group-item.disabled:hover .list-group-item-text,.list-group-item.disabled:focus .list-group-item-text{color:#777}.list-group-item.active,.list-group-item.active:hover,.list-group-item.active:focus{z-index:2;color:#fff;background-color:#337ab7;border-color:#337ab7}.list-group-item.active .list-group-item-heading,.list-group-item.active:hover .list-group-item-heading,.list-grou
p-item.active:focus .list-group-item-heading,.list-group-item.active .list-group-item-heading>small,.list-group-item.active:hover .list-group-item-heading>small,.list-group-item.active:focus .list-group-item-heading>small,.list-group-item.active .list-group-item-heading>.small,.list-group-item.active:hover .list-group-item-heading>.small,.list-group-item.active:focus .list-group-item-heading>.small{color:inherit}.list-group-item.active .list-group-item-text,.list-group-item.active:hover .list-group-item-text,.list-group-item.active:focus .list-group-item-text{color:#c7ddef}.list-group-item-success{color:#3c763d;background-color:#dff0d8}a.list-group-item-success,button.list-group-item-success{color:#3c763d}a.list-group-item-success .list-group-item-heading,button.list-group-item-success .list-group-item-heading{color:inherit}a.list-group-item-success:hover,button.list-group-item-success:hover,a.list-group-item-success:focus,button.list-group-item-success:focus{color:#3c763d;backgroun
d-color:#d0e9c6}a.list-group-item-success.active,button.list-group-item-success.active,a.list-group-item-success.active:hover,button.list-group-item-success.active:hover,a.list-group-item-success.active:focus,button.list-group-item-success.active:focus{color:#fff;background-color:#3c763d;border-color:#3c763d}.list-group-item-info{color:#31708f;background-color:#d9edf7}a.list-group-item-info,button.list-group-item-info{color:#31708f}a.list-group-item-info .list-group-item-heading,button.list-group-item-info .list-group-item-heading{color:inherit}a.list-group-item-info:hover,button.list-group-item-info:hover,a.list-group-item-info:focus,button.list-group-item-info:focus{color:#31708f;background-color:#c4e3f3}a.list-group-item-info.active,button.list-group-item-info.active,a.list-group-item-info.active:hover,button.list-group-item-info.active:hover,a.list-group-item-info.active:focus,button.list-group-item-info.active:focus{color:#fff;background-color:#31708f;border-color:#31708f}.list
-group-item-warning{color:#8a6d3b;background-color:#fcf8e3}a.list-group-item-warning,button.list-group-item-warning{color:#8a6d3b}a.list-group-item-warning .list-group-item-heading,button.list-group-item-warning .list-group-item-heading{color:inherit}a.list-group-item-warning:hover,button.list-group-item-warning:hover,a.list-group-item-warning:focus,button.list-group-item-warning:focus{color:#8a6d3b;background-color:#faf2cc}a.list-group-item-warning.active,button.list-group-item-warning.active,a.list-group-item-warning.active:hover,button.list-group-item-warning.active:hover,a.list-group-item-warning.active:focus,button.list-group-item-warning.active:focus{color:#fff;background-color:#8a6d3b;border-color:#8a6d3b}.list-group-item-danger{color:#a94442;background-color:#f2dede}a.list-group-item-danger,button.list-group-item-danger{color:#a94442}a.list-group-item-danger .list-group-item-heading,button.list-group-item-danger .list-group-item-heading{color:inherit}a.list-group-item-danger
:hover,button.list-group-item-danger:hover,a.list-group-item-danger:focus,button.list-group-item-danger:focus{color:#a94442;background-color:#ebcccc}a.list-group-item-danger.active,button.list-group-item-danger.active,a.list-group-item-danger.active:hover,button.list-group-item-danger.active:hover,a.list-group-item-danger.active:focus,button.list-group-item-danger.active:focus{color:#fff;background-color:#a94442;border-color:#a94442}.list-group-item-heading{margin-top:0;margin-bottom:5px}.list-group-item-text{margin-bottom:0;line-height:1.3}.panel{margin-bottom:20px;background-color:#fff;border:1px solid transparent;border-radius:4px;-webkit-box-shadow:0 1px 1px rgba(0,0,0,0.05);box-shadow:0 1px 1px rgba(0,0,0,0.05)}.panel-body{padding:15px}.panel-heading{padding:10px 15px;border-bottom:1px solid transparent;border-top-right-radius:3px;border-top-left-radius:3px}.panel-heading>.dropdown .dropdown-toggle{color:inherit}.panel-title{margin-top:0;margin-bottom:0;font-size:16px;color:inh
erit}.panel-title>a,.panel-title>small,.panel-title>.small,.panel-title>small>a,.panel-title>.small>a{color:inherit}.panel-footer{padding:10px 15px;background-color:#f5f5f5;border-top:1px solid #ddd;border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.list-group,.panel>.panel-collapse>.list-group{margin-bottom:0}.panel>.list-group .list-group-item,.panel>.panel-collapse>.list-group .list-group-item{border-width:1px 0;border-radius:0}.panel>.list-group:first-child .list-group-item:first-child,.panel>.panel-collapse>.list-group:first-child .list-group-item:first-child{border-top:0;border-top-right-radius:3px;border-top-left-radius:3px}.panel>.list-group:last-child .list-group-item:last-child,.panel>.panel-collapse>.list-group:last-child .list-group-item:last-child{border-bottom:0;border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.panel-heading+.panel-collapse>.list-group .list-group-item:first-child{border-top-right-radius:0;border-top-left-radius:0}.
panel-heading+.list-group .list-group-item:first-child{border-top-width:0}.list-group+.panel-footer{border-top-width:0}.panel>.table,.panel>.table-responsive>.table,.panel>.panel-collapse>.table{margin-bottom:0}.panel>.table caption,.panel>.table-responsive>.table caption,.panel>.panel-collapse>.table caption{padding-left:15px;padding-right:15px}.panel>.table:first-child,.panel>.table-responsive:first-child>.table:first-child{border-top-right-radius:3px;border-top-left-radius:3px}.panel>.table:first-child>thead:first-child>tr:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child{border-top-left-radius:3px;border-top-right-radius:3px}.panel>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-chil
d td:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table:first-child>thead:first-child>tr:first-child th:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:first-child{border-top-left-radius:3px}.panel>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table:first-child>thead:first-child>tr:first-child th:last-
child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:last-child{border-top-right-radius:3px}.panel>.table:last-child,.panel>.table-responsive:last-child>.table:last-child{border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.table:last-child>tbody:last-child>tr:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child{border-bottom-left-radius:3px;border-bottom-right-radius:3px}.panel>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table:last-child>tfoot
:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:first-child{border-bottom-left-radius:3px}.panel>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:la
st-child th:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:last-child{border-bottom-right-radius:3px}.panel>.panel-body+.table,.panel>.panel-body+.table-responsive,.panel>.table+.panel-body,.panel>.table-responsive+.panel-body{border-top:1px solid #ddd}.panel>.table>tbody:first-child>tr:first-child th,.panel>.table>tbody:first-child>tr:first-child td{border-top:0}.panel>.table-bordered,.panel>.table-responsive>.table-bordered{border:0}.panel>.table-bordered>thead>tr>th:first-child,.panel>.table-responsive
<TRUNCATED>
[17/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/bootstrap.min.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/bootstrap.min.js b/server/src/main/resources/com/cloudera/livy/server/ui/static/bootstrap.min.js
deleted file mode 100644
index 9bcd2fc..0000000
--- a/server/src/main/resources/com/cloudera/livy/server/ui/static/bootstrap.min.js
+++ /dev/null
@@ -1,7 +0,0 @@
-/*!
- * Bootstrap v3.3.7 (http://getbootstrap.com)
- * Copyright 2011-2016 Twitter, Inc.
- * Licensed under the MIT license
- */
-if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");+function(a){"use strict";var b=a.fn.jquery.split(" ")[0].split(".");if(b[0]<2&&b[1]<9||1==b[0]&&9==b[1]&&b[2]<1||b[0]>3)throw new Error("Bootstrap's JavaScript requires jQuery version 1.9.1 or higher, but lower than version 4")}(jQuery),+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var c in b)if(void 0!==a.style[c])return{end:b[c]};return!1}a.fn.emulateTransitionEnd=function(b){var c=!1,d=this;a(this).one("bsTransitionEnd",function(){c=!0});var e=function(){c||a(d).trigger(a.support.transition.end)};return setTimeout(e,b),this},a(function(){a.support.transition=b(),a.support.transition&&(a.event.special.bsTransitionEnd={bindType:a.support.transition.end,delegateType:a.support.transition.end,handle:function(b){if(a(b
.target).is(this))return b.handleObj.handler.apply(this,arguments)}})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var c=a(this),e=c.data("bs.alert");e||c.data("bs.alert",e=new d(this)),"string"==typeof b&&e[b].call(c)})}var c='[data-dismiss="alert"]',d=function(b){a(b).on("click",c,this.close)};d.VERSION="3.3.7",d.TRANSITION_DURATION=150,d.prototype.close=function(b){function c(){g.detach().trigger("closed.bs.alert").remove()}var e=a(this),f=e.attr("data-target");f||(f=e.attr("href"),f=f&&f.replace(/.*(?=#[^\s]*$)/,""));var g=a("#"===f?[]:f);b&&b.preventDefault(),g.length||(g=e.closest(".alert")),g.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(g.removeClass("in"),a.support.transition&&g.hasClass("fade")?g.one("bsTransitionEnd",c).emulateTransitionEnd(d.TRANSITION_DURATION):c())};var e=a.fn.alert;a.fn.alert=b,a.fn.alert.Constructor=d,a.fn.alert.noConflict=function(){return a.fn.alert=e,this},a(document).on("click.bs.alert.data-api",c
,d.prototype.close)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.button"),f="object"==typeof b&&b;e||d.data("bs.button",e=new c(this,f)),"toggle"==b?e.toggle():b&&e.setState(b)})}var c=function(b,d){this.$element=a(b),this.options=a.extend({},c.DEFAULTS,d),this.isLoading=!1};c.VERSION="3.3.7",c.DEFAULTS={loadingText:"loading..."},c.prototype.setState=function(b){var c="disabled",d=this.$element,e=d.is("input")?"val":"html",f=d.data();b+="Text",null==f.resetText&&d.data("resetText",d[e]()),setTimeout(a.proxy(function(){d[e](null==f[b]?this.options[b]:f[b]),"loadingText"==b?(this.isLoading=!0,d.addClass(c).attr(c,c).prop(c,!0)):this.isLoading&&(this.isLoading=!1,d.removeClass(c).removeAttr(c).prop(c,!1))},this),0)},c.prototype.toggle=function(){var a=!0,b=this.$element.closest('[data-toggle="buttons"]');if(b.length){var c=this.$element.find("input");"radio"==c.prop("type")?(c.prop("checked")&&(a=!1),b.find(".active").removeCla
ss("active"),this.$element.addClass("active")):"checkbox"==c.prop("type")&&(c.prop("checked")!==this.$element.hasClass("active")&&(a=!1),this.$element.toggleClass("active")),c.prop("checked",this.$element.hasClass("active")),a&&c.trigger("change")}else this.$element.attr("aria-pressed",!this.$element.hasClass("active")),this.$element.toggleClass("active")};var d=a.fn.button;a.fn.button=b,a.fn.button.Constructor=c,a.fn.button.noConflict=function(){return a.fn.button=d,this},a(document).on("click.bs.button.data-api",'[data-toggle^="button"]',function(c){var d=a(c.target).closest(".btn");b.call(d,"toggle"),a(c.target).is('input[type="radio"], input[type="checkbox"]')||(c.preventDefault(),d.is("input,button")?d.trigger("focus"):d.find("input:visible,button:visible").first().trigger("focus"))}).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',function(b){a(b.target).closest(".btn").toggleClass("focus",/^focus(in)?$/.test(b.type))})}(jQuery),+function(a){"us
e strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.carousel"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof b&&b),g="string"==typeof b?b:f.slide;e||d.data("bs.carousel",e=new c(this,f)),"number"==typeof b?e.to(b):g?e[g]():f.interval&&e.pause().cycle()})}var c=function(b,c){this.$element=a(b),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=null,this.sliding=null,this.interval=null,this.$active=null,this.$items=null,this.options.keyboard&&this.$element.on("keydown.bs.carousel",a.proxy(this.keydown,this)),"hover"==this.options.pause&&!("ontouchstart"in document.documentElement)&&this.$element.on("mouseenter.bs.carousel",a.proxy(this.pause,this)).on("mouseleave.bs.carousel",a.proxy(this.cycle,this))};c.VERSION="3.3.7",c.TRANSITION_DURATION=600,c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0,keyboard:!0},c.prototype.keydown=function(a){if(!/input|textarea/i.test(a.target.tagName)){switch(a.which){case 37:this.prev();b
reak;case 39:this.next();break;default:return}a.preventDefault()}},c.prototype.cycle=function(b){return b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(a){return this.$items=a.parent().children(".item"),this.$items.index(a||this.$active)},c.prototype.getItemForDirection=function(a,b){var c=this.getItemIndex(b),d="prev"==a&&0===c||"next"==a&&c==this.$items.length-1;if(d&&!this.options.wrap)return b;var e="prev"==a?-1:1,f=(c+e)%this.$items.length;return this.$items.eq(f)},c.prototype.to=function(a){var b=this,c=this.getItemIndex(this.$active=this.$element.find(".item.active"));if(!(a>this.$items.length-1||a<0))return this.sliding?this.$element.one("slid.bs.carousel",function(){b.to(a)}):c==a?this.pause().cycle():this.slide(a>c?"next":"prev",this.$items.eq(a))},c.prototype.pause=function(b){return b||(this.paused=!0),thi
s.$element.find(".next, .prev").length&&a.support.transition&&(this.$element.trigger(a.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},c.prototype.next=function(){if(!this.sliding)return this.slide("next")},c.prototype.prev=function(){if(!this.sliding)return this.slide("prev")},c.prototype.slide=function(b,d){var e=this.$element.find(".item.active"),f=d||this.getItemForDirection(b,e),g=this.interval,h="next"==b?"left":"right",i=this;if(f.hasClass("active"))return this.sliding=!1;var j=f[0],k=a.Event("slide.bs.carousel",{relatedTarget:j,direction:h});if(this.$element.trigger(k),!k.isDefaultPrevented()){if(this.sliding=!0,g&&this.pause(),this.$indicators.length){this.$indicators.find(".active").removeClass("active");var l=a(this.$indicators.children()[this.getItemIndex(f)]);l&&l.addClass("active")}var m=a.Event("slid.bs.carousel",{relatedTarget:j,direction:h});return a.support.transition&&this.$element.hasClass("slide")?(f.addClass(b),f[0].offs
etWidth,e.addClass(h),f.addClass(h),e.one("bsTransitionEnd",function(){f.removeClass([b,h].join(" ")).addClass("active"),e.removeClass(["active",h].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger(m)},0)}).emulateTransitionEnd(c.TRANSITION_DURATION)):(e.removeClass("active"),f.addClass("active"),this.sliding=!1,this.$element.trigger(m)),g&&this.cycle(),this}};var d=a.fn.carousel;a.fn.carousel=b,a.fn.carousel.Constructor=c,a.fn.carousel.noConflict=function(){return a.fn.carousel=d,this};var e=function(c){var d,e=a(this),f=a(e.attr("data-target")||(d=e.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""));if(f.hasClass("carousel")){var g=a.extend({},f.data(),e.data()),h=e.attr("data-slide-to");h&&(g.interval=!1),b.call(f,g),h&&f.data("bs.carousel").to(h),c.preventDefault()}};a(document).on("click.bs.carousel.data-api","[data-slide]",e).on("click.bs.carousel.data-api","[data-slide-to]",e),a(window).on("load",function(){a('[data-ride="carousel"]').each(function(){var c=a(this);
b.call(c,c.data())})})}(jQuery),+function(a){"use strict";function b(b){var c,d=b.attr("data-target")||(c=b.attr("href"))&&c.replace(/.*(?=#[^\s]+$)/,"");return a(d)}function c(b){return this.each(function(){var c=a(this),e=c.data("bs.collapse"),f=a.extend({},d.DEFAULTS,c.data(),"object"==typeof b&&b);!e&&f.toggle&&/show|hide/.test(b)&&(f.toggle=!1),e||c.data("bs.collapse",e=new d(this,f)),"string"==typeof b&&e[b]()})}var d=function(b,c){this.$element=a(b),this.options=a.extend({},d.DEFAULTS,c),this.$trigger=a('[data-toggle="collapse"][href="#'+b.id+'"],[data-toggle="collapse"][data-target="#'+b.id+'"]'),this.transitioning=null,this.options.parent?this.$parent=this.getParent():this.addAriaAndCollapsedClass(this.$element,this.$trigger),this.options.toggle&&this.toggle()};d.VERSION="3.3.7",d.TRANSITION_DURATION=350,d.DEFAULTS={toggle:!0},d.prototype.dimension=function(){var a=this.$element.hasClass("width");return a?"width":"height"},d.prototype.show=function(){if(!this.transitioning&
&!this.$element.hasClass("in")){var b,e=this.$parent&&this.$parent.children(".panel").children(".in, .collapsing");if(!(e&&e.length&&(b=e.data("bs.collapse"),b&&b.transitioning))){var f=a.Event("show.bs.collapse");if(this.$element.trigger(f),!f.isDefaultPrevented()){e&&e.length&&(c.call(e,"hide"),b||e.data("bs.collapse",null));var g=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[g](0).attr("aria-expanded",!0),this.$trigger.removeClass("collapsed").attr("aria-expanded",!0),this.transitioning=1;var h=function(){this.$element.removeClass("collapsing").addClass("collapse in")[g](""),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return h.call(this);var i=a.camelCase(["scroll",g].join("-"));this.$element.one("bsTransitionEnd",a.proxy(h,this)).emulateTransitionEnd(d.TRANSITION_DURATION)[g](this.$element[0][i])}}}},d.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var b=a.Event("hide.b
s.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.dimension();this.$element[c](this.$element[c]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse in").attr("aria-expanded",!1),this.$trigger.addClass("collapsed").attr("aria-expanded",!1),this.transitioning=1;var e=function(){this.transitioning=0,this.$element.removeClass("collapsing").addClass("collapse").trigger("hidden.bs.collapse")};return a.support.transition?void this.$element[c](0).one("bsTransitionEnd",a.proxy(e,this)).emulateTransitionEnd(d.TRANSITION_DURATION):e.call(this)}}},d.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()},d.prototype.getParent=function(){return a(this.options.parent).find('[data-toggle="collapse"][data-parent="'+this.options.parent+'"]').each(a.proxy(function(c,d){var e=a(d);this.addAriaAndCollapsedClass(b(e),e)},this)).end()},d.prototype.addAriaAndCollapsedClass=function(a,b){var c=a.hasClass("in");a.attr("aria-expande
d",c),b.toggleClass("collapsed",!c).attr("aria-expanded",c)};var e=a.fn.collapse;a.fn.collapse=c,a.fn.collapse.Constructor=d,a.fn.collapse.noConflict=function(){return a.fn.collapse=e,this},a(document).on("click.bs.collapse.data-api",'[data-toggle="collapse"]',function(d){var e=a(this);e.attr("data-target")||d.preventDefault();var f=b(e),g=f.data("bs.collapse"),h=g?"toggle":e.data();c.call(f,h)})}(jQuery),+function(a){"use strict";function b(b){var c=b.attr("data-target");c||(c=b.attr("href"),c=c&&/#[A-Za-z]/.test(c)&&c.replace(/.*(?=#[^\s]*$)/,""));var d=c&&a(c);return d&&d.length?d:b.parent()}function c(c){c&&3===c.which||(a(e).remove(),a(f).each(function(){var d=a(this),e=b(d),f={relatedTarget:this};e.hasClass("open")&&(c&&"click"==c.type&&/input|textarea/i.test(c.target.tagName)&&a.contains(e[0],c.target)||(e.trigger(c=a.Event("hide.bs.dropdown",f)),c.isDefaultPrevented()||(d.attr("aria-expanded","false"),e.removeClass("open").trigger(a.Event("hidden.bs.dropdown",f)))))}))}funct
ion d(b){return this.each(function(){var c=a(this),d=c.data("bs.dropdown");d||c.data("bs.dropdown",d=new g(this)),"string"==typeof b&&d[b].call(c)})}var e=".dropdown-backdrop",f='[data-toggle="dropdown"]',g=function(b){a(b).on("click.bs.dropdown",this.toggle)};g.VERSION="3.3.7",g.prototype.toggle=function(d){var e=a(this);if(!e.is(".disabled, :disabled")){var f=b(e),g=f.hasClass("open");if(c(),!g){"ontouchstart"in document.documentElement&&!f.closest(".navbar-nav").length&&a(document.createElement("div")).addClass("dropdown-backdrop").insertAfter(a(this)).on("click",c);var h={relatedTarget:this};if(f.trigger(d=a.Event("show.bs.dropdown",h)),d.isDefaultPrevented())return;e.trigger("focus").attr("aria-expanded","true"),f.toggleClass("open").trigger(a.Event("shown.bs.dropdown",h))}return!1}},g.prototype.keydown=function(c){if(/(38|40|27|32)/.test(c.which)&&!/input|textarea/i.test(c.target.tagName)){var d=a(this);if(c.preventDefault(),c.stopPropagation(),!d.is(".disabled, :disabled")){v
ar e=b(d),g=e.hasClass("open");if(!g&&27!=c.which||g&&27==c.which)return 27==c.which&&e.find(f).trigger("focus"),d.trigger("click");var h=" li:not(.disabled):visible a",i=e.find(".dropdown-menu"+h);if(i.length){var j=i.index(c.target);38==c.which&&j>0&&j--,40==c.which&&j<i.length-1&&j++,~j||(j=0),i.eq(j).trigger("focus")}}}};var h=a.fn.dropdown;a.fn.dropdown=d,a.fn.dropdown.Constructor=g,a.fn.dropdown.noConflict=function(){return a.fn.dropdown=h,this},a(document).on("click.bs.dropdown.data-api",c).on("click.bs.dropdown.data-api",".dropdown form",function(a){a.stopPropagation()}).on("click.bs.dropdown.data-api",f,g.prototype.toggle).on("keydown.bs.dropdown.data-api",f,g.prototype.keydown).on("keydown.bs.dropdown.data-api",".dropdown-menu",g.prototype.keydown)}(jQuery),+function(a){"use strict";function b(b,d){return this.each(function(){var e=a(this),f=e.data("bs.modal"),g=a.extend({},c.DEFAULTS,e.data(),"object"==typeof b&&b);f||e.data("bs.modal",f=new c(this,g)),"string"==typeof b?
f[b](d):g.show&&f.show(d)})}var c=function(b,c){this.options=c,this.$body=a(document.body),this.$element=a(b),this.$dialog=this.$element.find(".modal-dialog"),this.$backdrop=null,this.isShown=null,this.originalBodyPad=null,this.scrollbarWidth=0,this.ignoreBackdropClick=!1,this.options.remote&&this.$element.find(".modal-content").load(this.options.remote,a.proxy(function(){this.$element.trigger("loaded.bs.modal")},this))};c.VERSION="3.3.7",c.TRANSITION_DURATION=300,c.BACKDROP_TRANSITION_DURATION=150,c.DEFAULTS={backdrop:!0,keyboard:!0,show:!0},c.prototype.toggle=function(a){return this.isShown?this.hide():this.show(a)},c.prototype.show=function(b){var d=this,e=a.Event("show.bs.modal",{relatedTarget:b});this.$element.trigger(e),this.isShown||e.isDefaultPrevented()||(this.isShown=!0,this.checkScrollbar(),this.setScrollbar(),this.$body.addClass("modal-open"),this.escape(),this.resize(),this.$element.on("click.dismiss.bs.modal",'[data-dismiss="modal"]',a.proxy(this.hide,this)),this.$dial
og.on("mousedown.dismiss.bs.modal",function(){d.$element.one("mouseup.dismiss.bs.modal",function(b){a(b.target).is(d.$element)&&(d.ignoreBackdropClick=!0)})}),this.backdrop(function(){var e=a.support.transition&&d.$element.hasClass("fade");d.$element.parent().length||d.$element.appendTo(d.$body),d.$element.show().scrollTop(0),d.adjustDialog(),e&&d.$element[0].offsetWidth,d.$element.addClass("in"),d.enforceFocus();var f=a.Event("shown.bs.modal",{relatedTarget:b});e?d.$dialog.one("bsTransitionEnd",function(){d.$element.trigger("focus").trigger(f)}).emulateTransitionEnd(c.TRANSITION_DURATION):d.$element.trigger("focus").trigger(f)}))},c.prototype.hide=function(b){b&&b.preventDefault(),b=a.Event("hide.bs.modal"),this.$element.trigger(b),this.isShown&&!b.isDefaultPrevented()&&(this.isShown=!1,this.escape(),this.resize(),a(document).off("focusin.bs.modal"),this.$element.removeClass("in").off("click.dismiss.bs.modal").off("mouseup.dismiss.bs.modal"),this.$dialog.off("mousedown.dismiss.bs.m
odal"),a.support.transition&&this.$element.hasClass("fade")?this.$element.one("bsTransitionEnd",a.proxy(this.hideModal,this)).emulateTransitionEnd(c.TRANSITION_DURATION):this.hideModal())},c.prototype.enforceFocus=function(){a(document).off("focusin.bs.modal").on("focusin.bs.modal",a.proxy(function(a){document===a.target||this.$element[0]===a.target||this.$element.has(a.target).length||this.$element.trigger("focus")},this))},c.prototype.escape=function(){this.isShown&&this.options.keyboard?this.$element.on("keydown.dismiss.bs.modal",a.proxy(function(a){27==a.which&&this.hide()},this)):this.isShown||this.$element.off("keydown.dismiss.bs.modal")},c.prototype.resize=function(){this.isShown?a(window).on("resize.bs.modal",a.proxy(this.handleUpdate,this)):a(window).off("resize.bs.modal")},c.prototype.hideModal=function(){var a=this;this.$element.hide(),this.backdrop(function(){a.$body.removeClass("modal-open"),a.resetAdjustments(),a.resetScrollbar(),a.$element.trigger("hidden.bs.modal")})
},c.prototype.removeBackdrop=function(){this.$backdrop&&this.$backdrop.remove(),this.$backdrop=null},c.prototype.backdrop=function(b){var d=this,e=this.$element.hasClass("fade")?"fade":"";if(this.isShown&&this.options.backdrop){var f=a.support.transition&&e;if(this.$backdrop=a(document.createElement("div")).addClass("modal-backdrop "+e).appendTo(this.$body),this.$element.on("click.dismiss.bs.modal",a.proxy(function(a){return this.ignoreBackdropClick?void(this.ignoreBackdropClick=!1):void(a.target===a.currentTarget&&("static"==this.options.backdrop?this.$element[0].focus():this.hide()))},this)),f&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in"),!b)return;f?this.$backdrop.one("bsTransitionEnd",b).emulateTransitionEnd(c.BACKDROP_TRANSITION_DURATION):b()}else if(!this.isShown&&this.$backdrop){this.$backdrop.removeClass("in");var g=function(){d.removeBackdrop(),b&&b()};a.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one("bsTransitionEnd",g).emulateTransiti
onEnd(c.BACKDROP_TRANSITION_DURATION):g()}else b&&b()},c.prototype.handleUpdate=function(){this.adjustDialog()},c.prototype.adjustDialog=function(){var a=this.$element[0].scrollHeight>document.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&a?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!a?this.scrollbarWidth:""})},c.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},c.prototype.checkScrollbar=function(){var a=window.innerWidth;if(!a){var b=document.documentElement.getBoundingClientRect();a=b.right-Math.abs(b.left)}this.bodyIsOverflowing=document.body.clientWidth<a,this.scrollbarWidth=this.measureScrollbar()},c.prototype.setScrollbar=function(){var a=parseInt(this.$body.css("padding-right")||0,10);this.originalBodyPad=document.body.style.paddingRight||"",this.bodyIsOverflowing&&this.$body.css("padding-right",a+this.scrollbarWidth)},c.prototype.resetScrollbar=function(){this.$body.css("padding
-right",this.originalBodyPad)},c.prototype.measureScrollbar=function(){var a=document.createElement("div");a.className="modal-scrollbar-measure",this.$body.append(a);var b=a.offsetWidth-a.clientWidth;return this.$body[0].removeChild(a),b};var d=a.fn.modal;a.fn.modal=b,a.fn.modal.Constructor=c,a.fn.modal.noConflict=function(){return a.fn.modal=d,this},a(document).on("click.bs.modal.data-api",'[data-toggle="modal"]',function(c){var d=a(this),e=d.attr("href"),f=a(d.attr("data-target")||e&&e.replace(/.*(?=#[^\s]+$)/,"")),g=f.data("bs.modal")?"toggle":a.extend({remote:!/#/.test(e)&&e},f.data(),d.data());d.is("a")&&c.preventDefault(),f.one("show.bs.modal",function(a){a.isDefaultPrevented()||f.one("hidden.bs.modal",function(){d.is(":visible")&&d.trigger("focus")})}),b.call(f,g,this)})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.tooltip"),f="object"==typeof b&&b;!e&&/destroy|hide/.test(b)||(e||d.data("bs.tooltip",e=new c(this,f)),"
string"==typeof b&&e[b]())})}var c=function(a,b){this.type=null,this.options=null,this.enabled=null,this.timeout=null,this.hoverState=null,this.$element=null,this.inState=null,this.init("tooltip",a,b)};c.VERSION="3.3.7",c.TRANSITION_DURATION=150,c.DEFAULTS={animation:!0,placement:"top",selector:!1,template:'<div class="tooltip" role="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0}},c.prototype.init=function(b,c,d){if(this.enabled=!0,this.type=b,this.$element=a(c),this.options=this.getOptions(d),this.$viewport=this.options.viewport&&a(a.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+
this.type+" on the window.document object!");for(var e=this.options.trigger.split(" "),f=e.length;f--;){var g=e[f];if("click"==g)this.$element.on("click."+this.type,this.options.selector,a.proxy(this.toggle,this));else if("manual"!=g){var h="hover"==g?"mouseenter":"focusin",i="hover"==g?"mouseleave":"focusout";this.$element.on(h+"."+this.type,this.options.selector,a.proxy(this.enter,this)),this.$element.on(i+"."+this.type,this.options.selector,a.proxy(this.leave,this))}}this.options.selector?this._options=a.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.getOptions=function(b){return b=a.extend({},this.getDefaults(),this.$element.data(),b),b.delay&&"number"==typeof b.delay&&(b.delay={show:b.delay,hide:b.delay}),b},c.prototype.getDelegateOptions=function(){var b={},c=this.getDefaults();return this._options&&a.each(this._options,function(a,d){c[a]!=d&&(b[a]=d)}),b},c.prototype.enter=function(b){v
ar c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusin"==b.type?"focus":"hover"]=!0),c.tip().hasClass("in")||"in"==c.hoverState?void(c.hoverState="in"):(clearTimeout(c.timeout),c.hoverState="in",c.options.delay&&c.options.delay.show?void(c.timeout=setTimeout(function(){"in"==c.hoverState&&c.show()},c.options.delay.show)):c.show())},c.prototype.isInStateTrue=function(){for(var a in this.inState)if(this.inState[a])return!0;return!1},c.prototype.leave=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);if(c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusout"==b.type?"focus":"hover"]=!1),!c.isInStateTrue())return clearTimeout(c.timeout),c.hoverState="out",c.option
s.delay&&c.options.delay.hide?void(c.timeout=setTimeout(function(){"out"==c.hoverState&&c.hide()},c.options.delay.hide)):c.hide()},c.prototype.show=function(){var b=a.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(b);var d=a.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(b.isDefaultPrevented()||!d)return;var e=this,f=this.tip(),g=this.getUID(this.type);this.setContent(),f.attr("id",g),this.$element.attr("aria-describedby",g),this.options.animation&&f.addClass("fade");var h="function"==typeof this.options.placement?this.options.placement.call(this,f[0],this.$element[0]):this.options.placement,i=/\s?auto?\s?/i,j=i.test(h);j&&(h=h.replace(i,"")||"top"),f.detach().css({top:0,left:0,display:"block"}).addClass(h).data("bs."+this.type,this),this.options.container?f.appendTo(this.options.container):f.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type);var k=this.getPosition(),l=f[0].offsetWidth,m=f[0
].offsetHeight;if(j){var n=h,o=this.getPosition(this.$viewport);h="bottom"==h&&k.bottom+m>o.bottom?"top":"top"==h&&k.top-m<o.top?"bottom":"right"==h&&k.right+l>o.width?"left":"left"==h&&k.left-l<o.left?"right":h,f.removeClass(n).addClass(h)}var p=this.getCalculatedOffset(h,k,l,m);this.applyPlacement(p,h);var q=function(){var a=e.hoverState;e.$element.trigger("shown.bs."+e.type),e.hoverState=null,"out"==a&&e.leave(e)};a.support.transition&&this.$tip.hasClass("fade")?f.one("bsTransitionEnd",q).emulateTransitionEnd(c.TRANSITION_DURATION):q()}},c.prototype.applyPlacement=function(b,c){var d=this.tip(),e=d[0].offsetWidth,f=d[0].offsetHeight,g=parseInt(d.css("margin-top"),10),h=parseInt(d.css("margin-left"),10);isNaN(g)&&(g=0),isNaN(h)&&(h=0),b.top+=g,b.left+=h,a.offset.setOffset(d[0],a.extend({using:function(a){d.css({top:Math.round(a.top),left:Math.round(a.left)})}},b),0),d.addClass("in");var i=d[0].offsetWidth,j=d[0].offsetHeight;"top"==c&&j!=f&&(b.top=b.top+f-j);var k=this.getViewport
AdjustedDelta(c,b,i,j);k.left?b.left+=k.left:b.top+=k.top;var l=/top|bottom/.test(c),m=l?2*k.left-e+i:2*k.top-f+j,n=l?"offsetWidth":"offsetHeight";d.offset(b),this.replaceArrow(m,d[0][n],l)},c.prototype.replaceArrow=function(a,b,c){this.arrow().css(c?"left":"top",50*(1-a/b)+"%").css(c?"top":"left","")},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle();a.find(".tooltip-inner")[this.options.html?"html":"text"](b),a.removeClass("fade in top bottom left right")},c.prototype.hide=function(b){function d(){"in"!=e.hoverState&&f.detach(),e.$element&&e.$element.removeAttr("aria-describedby").trigger("hidden.bs."+e.type),b&&b()}var e=this,f=a(this.$tip),g=a.Event("hide.bs."+this.type);if(this.$element.trigger(g),!g.isDefaultPrevented())return f.removeClass("in"),a.support.transition&&f.hasClass("fade")?f.one("bsTransitionEnd",d).emulateTransitionEnd(c.TRANSITION_DURATION):d(),this.hoverState=null,this},c.prototype.fixTitle=function(){var a=this.$element;(a.attr("title")||"s
tring"!=typeof a.attr("data-original-title"))&&a.attr("data-original-title",a.attr("title")||"").attr("title","")},c.prototype.hasContent=function(){return this.getTitle()},c.prototype.getPosition=function(b){b=b||this.$element;var c=b[0],d="BODY"==c.tagName,e=c.getBoundingClientRect();null==e.width&&(e=a.extend({},e,{width:e.right-e.left,height:e.bottom-e.top}));var f=window.SVGElement&&c instanceof window.SVGElement,g=d?{top:0,left:0}:f?null:b.offset(),h={scroll:d?document.documentElement.scrollTop||document.body.scrollTop:b.scrollTop()},i=d?{width:a(window).width(),height:a(window).height()}:null;return a.extend({},e,h,i,g)},c.prototype.getCalculatedOffset=function(a,b,c,d){return"bottom"==a?{top:b.top+b.height,left:b.left+b.width/2-c/2}:"top"==a?{top:b.top-d,left:b.left+b.width/2-c/2}:"left"==a?{top:b.top+b.height/2-d/2,left:b.left-c}:{top:b.top+b.height/2-d/2,left:b.left+b.width}},c.prototype.getViewportAdjustedDelta=function(a,b,c,d){var e={top:0,left:0};if(!this.$viewport)ret
urn e;var f=this.options.viewport&&this.options.viewport.padding||0,g=this.getPosition(this.$viewport);if(/right|left/.test(a)){var h=b.top-f-g.scroll,i=b.top+f-g.scroll+d;h<g.top?e.top=g.top-h:i>g.top+g.height&&(e.top=g.top+g.height-i)}else{var j=b.left-f,k=b.left+f+c;j<g.left?e.left=g.left-j:k>g.right&&(e.left=g.left+g.width-k)}return e},c.prototype.getTitle=function(){var a,b=this.$element,c=this.options;return a=b.attr("data-original-title")||("function"==typeof c.title?c.title.call(b[0]):c.title)},c.prototype.getUID=function(a){do a+=~~(1e6*Math.random());while(document.getElementById(a));return a},c.prototype.tip=function(){if(!this.$tip&&(this.$tip=a(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},c.prototype.enable=function(){this.enabled=!0},c.prototype.disable=function(){th
is.enabled=!1},c.prototype.toggleEnabled=function(){this.enabled=!this.enabled},c.prototype.toggle=function(b){var c=this;b&&(c=a(b.currentTarget).data("bs."+this.type),c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c))),b?(c.inState.click=!c.inState.click,c.isInStateTrue()?c.enter(c):c.leave(c)):c.tip().hasClass("in")?c.leave(c):c.enter(c)},c.prototype.destroy=function(){var a=this;clearTimeout(this.timeout),this.hide(function(){a.$element.off("."+a.type).removeData("bs."+a.type),a.$tip&&a.$tip.detach(),a.$tip=null,a.$arrow=null,a.$viewport=null,a.$element=null})};var d=a.fn.tooltip;a.fn.tooltip=b,a.fn.tooltip.Constructor=c,a.fn.tooltip.noConflict=function(){return a.fn.tooltip=d,this}}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.popover"),f="object"==typeof b&&b;!e&&/destroy|hide/.test(b)||(e||d.data("bs.popover",e=new c(this,f)),"string"==typeof b&&e[b]())})}v
ar c=function(a,b){this.init("popover",a,b)};if(!a.fn.tooltip)throw new Error("Popover requires tooltip.js");c.VERSION="3.3.7",c.DEFAULTS=a.extend({},a.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:'<div class="popover" role="tooltip"><div class="arrow"></div><h3 class="popover-title"></h3><div class="popover-content"></div></div>'}),c.prototype=a.extend({},a.fn.tooltip.Constructor.prototype),c.prototype.constructor=c,c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle(),c=this.getContent();a.find(".popover-title")[this.options.html?"html":"text"](b),a.find(".popover-content").children().detach().end()[this.options.html?"string"==typeof c?"html":"append":"text"](c),a.removeClass("fade top bottom left right in"),a.find(".popover-title").html()||a.find(".popover-title").hide()},c.prototype.hasContent=function(){return this.getTitle()||this.getContent()},c.prototype.getContent=
function(){var a=this.$element,b=this.options;return a.attr("data-content")||("function"==typeof b.content?b.content.call(a[0]):b.content)},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var d=a.fn.popover;a.fn.popover=b,a.fn.popover.Constructor=c,a.fn.popover.noConflict=function(){return a.fn.popover=d,this}}(jQuery),+function(a){"use strict";function b(c,d){this.$body=a(document.body),this.$scrollElement=a(a(c).is(document.body)?window:c),this.options=a.extend({},b.DEFAULTS,d),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",a.proxy(this.process,this)),this.refresh(),this.process()}function c(c){return this.each(function(){var d=a(this),e=d.data("bs.scrollspy"),f="object"==typeof c&&c;e||d.data("bs.scrollspy",e=new b(this,f)),"string"==typeof c&&e[c]()})}b.VERSION="3.3.7",b.DEFAULTS={offset:10},b.prototype.getScrollH
eight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},b.prototype.refresh=function(){var b=this,c="offset",d=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),a.isWindow(this.$scrollElement[0])||(c="position",d=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var b=a(this),e=b.data("target")||b.attr("href"),f=/^#./.test(e)&&a(e);return f&&f.length&&f.is(":visible")&&[[f[c]().top+d,e]]||null}).sort(function(a,b){return a[0]-b[0]}).each(function(){b.offsets.push(this[0]),b.targets.push(this[1])})},b.prototype.process=function(){var a,b=this.$scrollElement.scrollTop()+this.options.offset,c=this.getScrollHeight(),d=this.options.offset+c-this.$scrollElement.height(),e=this.offsets,f=this.targets,g=this.activeTarget;if(this.scrollHeight!=c&&this.refresh(),b>=d)return g!=(a=f[f.length-1])&&this.activate(a);if(g&&b<e[0])return this.activeTarget=null,this.
clear();for(a=e.length;a--;)g!=f[a]&&b>=e[a]&&(void 0===e[a+1]||b<e[a+1])&&this.activate(f[a])},b.prototype.activate=function(b){
-this.activeTarget=b,this.clear();var c=this.selector+'[data-target="'+b+'"],'+this.selector+'[href="'+b+'"]',d=a(c).parents("li").addClass("active");d.parent(".dropdown-menu").length&&(d=d.closest("li.dropdown").addClass("active")),d.trigger("activate.bs.scrollspy")},b.prototype.clear=function(){a(this.selector).parentsUntil(this.options.target,".active").removeClass("active")};var d=a.fn.scrollspy;a.fn.scrollspy=c,a.fn.scrollspy.Constructor=b,a.fn.scrollspy.noConflict=function(){return a.fn.scrollspy=d,this},a(window).on("load.bs.scrollspy.data-api",function(){a('[data-spy="scroll"]').each(function(){var b=a(this);c.call(b,b.data())})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.tab");e||d.data("bs.tab",e=new c(this)),"string"==typeof b&&e[b]()})}var c=function(b){this.element=a(b)};c.VERSION="3.3.7",c.TRANSITION_DURATION=150,c.prototype.show=function(){var b=this.element,c=b.closest("ul:not(.dropdown-menu)"),d=b.data("ta
rget");if(d||(d=b.attr("href"),d=d&&d.replace(/.*(?=#[^\s]*$)/,"")),!b.parent("li").hasClass("active")){var e=c.find(".active:last a"),f=a.Event("hide.bs.tab",{relatedTarget:b[0]}),g=a.Event("show.bs.tab",{relatedTarget:e[0]});if(e.trigger(f),b.trigger(g),!g.isDefaultPrevented()&&!f.isDefaultPrevented()){var h=a(d);this.activate(b.closest("li"),c),this.activate(h,h.parent(),function(){e.trigger({type:"hidden.bs.tab",relatedTarget:b[0]}),b.trigger({type:"shown.bs.tab",relatedTarget:e[0]})})}}},c.prototype.activate=function(b,d,e){function f(){g.removeClass("active").find("> .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),b.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),h?(b[0].offsetWidth,b.addClass("in")):b.removeClass("fade"),b.parent(".dropdown-menu").length&&b.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),e&&e()}var g=d.find("> .active"),h=e&
&a.support.transition&&(g.length&&g.hasClass("fade")||!!d.find("> .fade").length);g.length&&h?g.one("bsTransitionEnd",f).emulateTransitionEnd(c.TRANSITION_DURATION):f(),g.removeClass("in")};var d=a.fn.tab;a.fn.tab=b,a.fn.tab.Constructor=c,a.fn.tab.noConflict=function(){return a.fn.tab=d,this};var e=function(c){c.preventDefault(),b.call(a(this),"show")};a(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',e).on("click.bs.tab.data-api",'[data-toggle="pill"]',e)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.affix"),f="object"==typeof b&&b;e||d.data("bs.affix",e=new c(this,f)),"string"==typeof b&&e[b]()})}var c=function(b,d){this.options=a.extend({},c.DEFAULTS,d),this.$target=a(this.options.target).on("scroll.bs.affix.data-api",a.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",a.proxy(this.checkPositionWithEventLoop,this)),this.$element=a(b),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkP
osition()};c.VERSION="3.3.7",c.RESET="affix affix-top affix-bottom",c.DEFAULTS={offset:0,target:window},c.prototype.getState=function(a,b,c,d){var e=this.$target.scrollTop(),f=this.$element.offset(),g=this.$target.height();if(null!=c&&"top"==this.affixed)return e<c&&"top";if("bottom"==this.affixed)return null!=c?!(e+this.unpin<=f.top)&&"bottom":!(e+g<=a-d)&&"bottom";var h=null==this.affixed,i=h?e:f.top,j=h?g:b;return null!=c&&e<=c?"top":null!=d&&i+j>=a-d&&"bottom"},c.prototype.getPinnedOffset=function(){if(this.pinnedOffset)return this.pinnedOffset;this.$element.removeClass(c.RESET).addClass("affix");var a=this.$target.scrollTop(),b=this.$element.offset();return this.pinnedOffset=b.top-a},c.prototype.checkPositionWithEventLoop=function(){setTimeout(a.proxy(this.checkPosition,this),1)},c.prototype.checkPosition=function(){if(this.$element.is(":visible")){var b=this.$element.height(),d=this.options.offset,e=d.top,f=d.bottom,g=Math.max(a(document).height(),a(document.body).height());"o
bject"!=typeof d&&(f=e=d),"function"==typeof e&&(e=d.top(this.$element)),"function"==typeof f&&(f=d.bottom(this.$element));var h=this.getState(g,b,e,f);if(this.affixed!=h){null!=this.unpin&&this.$element.css("top","");var i="affix"+(h?"-"+h:""),j=a.Event(i+".bs.affix");if(this.$element.trigger(j),j.isDefaultPrevented())return;this.affixed=h,this.unpin="bottom"==h?this.getPinnedOffset():null,this.$element.removeClass(c.RESET).addClass(i).trigger(i.replace("affix","affixed")+".bs.affix")}"bottom"==h&&this.$element.offset({top:g-b-f})}};var d=a.fn.affix;a.fn.affix=b,a.fn.affix.Constructor=c,a.fn.affix.noConflict=function(){return a.fn.affix=d,this},a(window).on("load",function(){a('[data-spy="affix"]').each(function(){var c=a(this),d=c.data();d.offset=d.offset||{},null!=d.offsetBottom&&(d.offset.bottom=d.offsetBottom),null!=d.offsetTop&&(d.offset.top=d.offsetTop),b.call(c,d)})})}(jQuery);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/dataTables.bootstrap.min.css
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/dataTables.bootstrap.min.css b/server/src/main/resources/com/cloudera/livy/server/ui/static/dataTables.bootstrap.min.css
deleted file mode 100644
index 66a70ab..0000000
--- a/server/src/main/resources/com/cloudera/livy/server/ui/static/dataTables.bootstrap.min.css
+++ /dev/null
@@ -1 +0,0 @@
-table.dataTable{clear:both;margin-top:6px !important;margin-bottom:6px !important;max-width:none !important;border-collapse:separate !important}table.dataTable td,table.dataTable th{-webkit-box-sizing:content-box;box-sizing:content-box}table.dataTable td.dataTables_empty,table.dataTable th.dataTables_empty{text-align:center}table.dataTable.nowrap th,table.dataTable.nowrap td{white-space:nowrap}div.dataTables_wrapper div.dataTables_length label{font-weight:normal;text-align:left;white-space:nowrap}div.dataTables_wrapper div.dataTables_length select{width:75px;display:inline-block}div.dataTables_wrapper div.dataTables_filter{text-align:right}div.dataTables_wrapper div.dataTables_filter label{font-weight:normal;white-space:nowrap;text-align:left}div.dataTables_wrapper div.dataTables_filter input{margin-left:0.5em;display:inline-block;width:auto}div.dataTables_wrapper div.dataTables_info{padding-top:8px;white-space:nowrap}div.dataTables_wrapper div.dataTables_paginate{margin:0;white-spa
ce:nowrap;text-align:right}div.dataTables_wrapper div.dataTables_paginate ul.pagination{margin:2px 0;white-space:nowrap}div.dataTables_wrapper div.dataTables_processing{position:absolute;top:50%;left:50%;width:200px;margin-left:-100px;margin-top:-26px;text-align:center;padding:1em 0}table.dataTable thead>tr>th.sorting_asc,table.dataTable thead>tr>th.sorting_desc,table.dataTable thead>tr>th.sorting,table.dataTable thead>tr>td.sorting_asc,table.dataTable thead>tr>td.sorting_desc,table.dataTable thead>tr>td.sorting{padding-right:30px}table.dataTable thead>tr>th:active,table.dataTable thead>tr>td:active{outline:none}table.dataTable thead .sorting,table.dataTable thead .sorting_asc,table.dataTable thead .sorting_desc,table.dataTable thead .sorting_asc_disabled,table.dataTable thead .sorting_desc_disabled{cursor:pointer;position:relative}table.dataTable thead .sorting:after,table.dataTable thead .sorting_asc:after,table.dataTable thead .sorting_desc:after,table.dataTable thead .sorting_as
c_disabled:after,table.dataTable thead .sorting_desc_disabled:after{position:absolute;bottom:8px;right:8px;display:block;font-family:'Glyphicons Halflings';opacity:0.5}table.dataTable thead .sorting:after{opacity:0.2;content:"\e150"}table.dataTable thead .sorting_asc:after{content:"\e155"}table.dataTable thead .sorting_desc:after{content:"\e156"}table.dataTable thead .sorting_asc_disabled:after,table.dataTable thead .sorting_desc_disabled:after{color:#eee}div.dataTables_scrollHead table.dataTable{margin-bottom:0 !important}div.dataTables_scrollBody>table{border-top:none;margin-top:0 !important;margin-bottom:0 !important}div.dataTables_scrollBody>table>thead .sorting:after,div.dataTables_scrollBody>table>thead .sorting_asc:after,div.dataTables_scrollBody>table>thead .sorting_desc:after{display:none}div.dataTables_scrollBody>table>tbody>tr:first-child>th,div.dataTables_scrollBody>table>tbody>tr:first-child>td{border-top:none}div.dataTables_scrollFoot>table{margin-top:0 !important;bord
er-top:none}@media screen and (max-width: 767px){div.dataTables_wrapper div.dataTables_length,div.dataTables_wrapper div.dataTables_filter,div.dataTables_wrapper div.dataTables_info,div.dataTables_wrapper div.dataTables_paginate{text-align:center}}table.dataTable.table-condensed>thead>tr>th{padding-right:20px}table.dataTable.table-condensed .sorting:after,table.dataTable.table-condensed .sorting_asc:after,table.dataTable.table-condensed .sorting_desc:after{top:6px;right:6px}table.table-bordered.dataTable th,table.table-bordered.dataTable td{border-left-width:0}table.table-bordered.dataTable th:last-child,table.table-bordered.dataTable th:last-child,table.table-bordered.dataTable td:last-child,table.table-bordered.dataTable td:last-child{border-right-width:0}table.table-bordered.dataTable tbody th,table.table-bordered.dataTable tbody td{border-bottom-width:0}div.dataTables_scrollHead table.table-bordered{border-bottom-width:0}div.table-responsive>div.dataTables_wrapper>div.row{margin
:0}div.table-responsive>div.dataTables_wrapper>div.row>div[class^="col-"]:first-child{padding-left:0}div.table-responsive>div.dataTables_wrapper>div.row>div[class^="col-"]:last-child{padding-right:0}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/dataTables.bootstrap.min.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/dataTables.bootstrap.min.js b/server/src/main/resources/com/cloudera/livy/server/ui/static/dataTables.bootstrap.min.js
deleted file mode 100644
index 98661c6..0000000
--- a/server/src/main/resources/com/cloudera/livy/server/ui/static/dataTables.bootstrap.min.js
+++ /dev/null
@@ -1,8 +0,0 @@
-/*!
- DataTables Bootstrap 3 integration
- ©2011-2015 SpryMedia Ltd - datatables.net/license
-*/
-(function(b){"function"===typeof define&&define.amd?define(["jquery","datatables.net"],function(a){return b(a,window,document)}):"object"===typeof exports?module.exports=function(a,d){a||(a=window);if(!d||!d.fn.dataTable)d=require("datatables.net")(a,d).$;return b(d,a,a.document)}:b(jQuery,window,document)})(function(b,a,d,m){var f=b.fn.dataTable;b.extend(!0,f.defaults,{dom:"<'row'<'col-sm-6'l><'col-sm-6'f>><'row'<'col-sm-12'tr>><'row'<'col-sm-5'i><'col-sm-7'p>>",renderer:"bootstrap"});b.extend(f.ext.classes,
-{sWrapper:"dataTables_wrapper form-inline dt-bootstrap",sFilterInput:"form-control input-sm",sLengthSelect:"form-control input-sm",sProcessing:"dataTables_processing panel panel-default"});f.ext.renderer.pageButton.bootstrap=function(a,h,r,s,j,n){var o=new f.Api(a),t=a.oClasses,k=a.oLanguage.oPaginate,u=a.oLanguage.oAria.paginate||{},e,g,p=0,q=function(d,f){var l,h,i,c,m=function(a){a.preventDefault();!b(a.currentTarget).hasClass("disabled")&&o.page()!=a.data.action&&o.page(a.data.action).draw("page")};
-l=0;for(h=f.length;l<h;l++)if(c=f[l],b.isArray(c))q(d,c);else{g=e="";switch(c){case "ellipsis":e="…";g="disabled";break;case "first":e=k.sFirst;g=c+(0<j?"":" disabled");break;case "previous":e=k.sPrevious;g=c+(0<j?"":" disabled");break;case "next":e=k.sNext;g=c+(j<n-1?"":" disabled");break;case "last":e=k.sLast;g=c+(j<n-1?"":" disabled");break;default:e=c+1,g=j===c?"active":""}e&&(i=b("<li>",{"class":t.sPageButton+" "+g,id:0===r&&"string"===typeof c?a.sTableId+"_"+c:null}).append(b("<a>",{href:"#",
-"aria-controls":a.sTableId,"aria-label":u[c],"data-dt-idx":p,tabindex:a.iTabIndex}).html(e)).appendTo(d),a.oApi._fnBindAction(i,{action:c},m),p++)}},i;try{i=b(h).find(d.activeElement).data("dt-idx")}catch(v){}q(b(h).empty().html('<ul class="pagination"/>').children("ul"),s);i!==m&&b(h).find("[data-dt-idx="+i+"]").focus()};return f});
[11/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/jquery-3.2.1.min.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/jquery-3.2.1.min.js b/server/src/main/resources/org/apache/livy/server/ui/static/jquery-3.2.1.min.js
new file mode 100644
index 0000000..644d35e
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/jquery-3.2.1.min.js
@@ -0,0 +1,4 @@
+/*! jQuery v3.2.1 | (c) JS Foundation and other contributors | jquery.org/license */
+!function(a,b){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){"use strict";var c=[],d=a.document,e=Object.getPrototypeOf,f=c.slice,g=c.concat,h=c.push,i=c.indexOf,j={},k=j.toString,l=j.hasOwnProperty,m=l.toString,n=m.call(Object),o={};function p(a,b){b=b||d;var c=b.createElement("script");c.text=a,b.head.appendChild(c).parentNode.removeChild(c)}var q="3.2.1",r=function(a,b){return new r.fn.init(a,b)},s=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,t=/^-ms-/,u=/-([a-z])/g,v=function(a,b){return b.toUpperCase()};r.fn=r.prototype={jquery:q,constructor:r,length:0,toArray:function(){return f.call(this)},get:function(a){return null==a?f.call(this):a<0?this[a+this.length]:this[a]},pushStack:function(a){var b=r.merge(this.constructor(),a);return b.prevObject=this,b},each:function(a){retu
rn r.each(this,a)},map:function(a){return this.pushStack(r.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(f.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(a<0?b:0);return this.pushStack(c>=0&&c<b?[this[c]]:[])},end:function(){return this.prevObject||this.constructor()},push:h,sort:c.sort,splice:c.splice},r.extend=r.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||r.isFunction(g)||(g={}),h===i&&(g=this,h--);h<i;h++)if(null!=(a=arguments[h]))for(b in a)c=g[b],d=a[b],g!==d&&(j&&d&&(r.isPlainObject(d)||(e=Array.isArray(d)))?(e?(e=!1,f=c&&Array.isArray(c)?c:[]):f=c&&r.isPlainObject(c)?c:{},g[b]=r.extend(j,f,d)):void 0!==d&&(g[b]=d));return g},r.extend({expando:"jQuery"+(q+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},no
op:function(){},isFunction:function(a){return"function"===r.type(a)},isWindow:function(a){return null!=a&&a===a.window},isNumeric:function(a){var b=r.type(a);return("number"===b||"string"===b)&&!isNaN(a-parseFloat(a))},isPlainObject:function(a){var b,c;return!(!a||"[object Object]"!==k.call(a))&&(!(b=e(a))||(c=l.call(b,"constructor")&&b.constructor,"function"==typeof c&&m.call(c)===n))},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?j[k.call(a)]||"object":typeof a},globalEval:function(a){p(a)},camelCase:function(a){return a.replace(t,"ms-").replace(u,v)},each:function(a,b){var c,d=0;if(w(a)){for(c=a.length;d<c;d++)if(b.call(a[d],d,a[d])===!1)break}else for(d in a)if(b.call(a[d],d,a[d])===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(s,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(w(Object(a))?r.merge(c,"string"==typeof a?[a]:a):h.call(c,a)),c},inArray:funct
ion(a,b,c){return null==b?-1:i.call(b,a,c)},merge:function(a,b){for(var c=+b.length,d=0,e=a.length;d<c;d++)a[e++]=b[d];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;f<g;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,e,f=0,h=[];if(w(a))for(d=a.length;f<d;f++)e=b(a[f],f,c),null!=e&&h.push(e);else for(f in a)e=b(a[f],f,c),null!=e&&h.push(e);return g.apply([],h)},guid:1,proxy:function(a,b){var c,d,e;if("string"==typeof b&&(c=a[b],b=a,a=c),r.isFunction(a))return d=f.call(arguments,2),e=function(){return a.apply(b||this,d.concat(f.call(arguments)))},e.guid=a.guid=a.guid||r.guid++,e},now:Date.now,support:o}),"function"==typeof Symbol&&(r.fn[Symbol.iterator]=c[Symbol.iterator]),r.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(a,b){j["[object "+b+"]"]=b.toLowerCase()});function w(a){var b=!!a&&"length"in a&&a.length,c=r.type(a);return"function"!==c&&!r.isWindow(a)&&("array"===c||0===b||"
number"==typeof b&&b>0&&b-1 in a)}var x=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=ha(),z=ha(),A=ha(),B=function(a,b){return a===b&&(l=!0),0},C={}.hasOwnProperty,D=[],E=D.pop,F=D.push,G=D.push,H=D.slice,I=function(a,b){for(var c=0,d=a.length;c<d;c++)if(a[c]===b)return c;return-1},J="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",K="[\\x20\\t\\r\\n\\f]",L="(?:\\\\.|[\\w-]|[^\0-\\xa0])+",M="\\["+K+"*("+L+")(?:"+K+"*([*^$|!~]?=)"+K+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+L+"))|)"+K+"*\\]",N=":("+L+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+M+")*)|.*)\\)|)",O=new RegExp(K+"+","g"),P=new RegExp("^"+K+"+|((?:^|[^\\\\])(?:\\\\.)*)"+K+"+$","g"),Q=new RegExp("^"+K+"*,"+K+"*"),R=new RegExp("^"+K+"*([>+~]|"+K+")"+K+"*"),S=new RegExp("="+K+"*([^\\]'\"]*?)"+K+"*\\]","g"),T=new RegExp(N),U=new RegExp("^
"+L+"$"),V={ID:new RegExp("^#("+L+")"),CLASS:new RegExp("^\\.("+L+")"),TAG:new RegExp("^("+L+"|[*])"),ATTR:new RegExp("^"+M),PSEUDO:new RegExp("^"+N),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+K+"*(even|odd|(([+-]|)(\\d*)n|)"+K+"*(?:([+-]|)"+K+"*(\\d+)|))"+K+"*\\)|)","i"),bool:new RegExp("^(?:"+J+")$","i"),needsContext:new RegExp("^"+K+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+K+"*((?:-\\d)?\\d*)"+K+"*\\)|)(?=[^-]|$)","i")},W=/^(?:input|select|textarea|button)$/i,X=/^h\d$/i,Y=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,$=/[+~]/,_=new RegExp("\\\\([\\da-f]{1,6}"+K+"?|("+K+")|.)","ig"),aa=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:d<0?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},ba=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ca=function(a,b){return b?"\0"===a?"\ufffd":a.slice(0,-1)+"\\"+a.charCodeAt(a.length-1).toString(16)+" ":"\\"+a},da=function(){m()},ea=ta(function(a){retur
n a.disabled===!0&&("form"in a||"label"in a)},{dir:"parentNode",next:"legend"});try{G.apply(D=H.call(v.childNodes),v.childNodes),D[v.childNodes.length].nodeType}catch(fa){G={apply:D.length?function(a,b){F.apply(a,H.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function ga(a,b,d,e){var f,h,j,k,l,o,r,s=b&&b.ownerDocument,w=b?b.nodeType:9;if(d=d||[],"string"!=typeof a||!a||1!==w&&9!==w&&11!==w)return d;if(!e&&((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,p)){if(11!==w&&(l=Z.exec(a)))if(f=l[1]){if(9===w){if(!(j=b.getElementById(f)))return d;if(j.id===f)return d.push(j),d}else if(s&&(j=s.getElementById(f))&&t(b,j)&&j.id===f)return d.push(j),d}else{if(l[2])return G.apply(d,b.getElementsByTagName(a)),d;if((f=l[3])&&c.getElementsByClassName&&b.getElementsByClassName)return G.apply(d,b.getElementsByClassName(f)),d}if(c.qsa&&!A[a+" "]&&(!q||!q.test(a))){if(1!==w)s=b,r=a;else if("object"!==b.nodeName.toLowerCase()){(k=b.getAttribute("id"))?k=k.replace(ba,ca):b.set
Attribute("id",k=u),o=g(a),h=o.length;while(h--)o[h]="#"+k+" "+sa(o[h]);r=o.join(","),s=$.test(a)&&qa(b.parentNode)||b}if(r)try{return G.apply(d,s.querySelectorAll(r)),d}catch(x){}finally{k===u&&b.removeAttribute("id")}}}return i(a.replace(P,"$1"),b,d,e)}function ha(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ia(a){return a[u]=!0,a}function ja(a){var b=n.createElement("fieldset");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function ka(a,b){var c=a.split("|"),e=c.length;while(e--)d.attrHandle[c[e]]=b}function la(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&a.sourceIndex-b.sourceIndex;if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function ma(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function na(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}f
unction oa(a){return function(b){return"form"in b?b.parentNode&&b.disabled===!1?"label"in b?"label"in b.parentNode?b.parentNode.disabled===a:b.disabled===a:b.isDisabled===a||b.isDisabled!==!a&&ea(b)===a:b.disabled===a:"label"in b&&b.disabled===a}}function pa(a){return ia(function(b){return b=+b,ia(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function qa(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=ga.support={},f=ga.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return!!b&&"HTML"!==b.nodeName},m=ga.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=n.documentElement,p=!f(n),v!==n&&(e=n.defaultView)&&e.top!==e&&(e.addEventListener?e.addEventListener("unload",da,!1):e.attachEvent&&e.attachEvent("onunload",da)),c.attributes=ja(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=ja(function(a){return a.append
Child(n.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=Y.test(n.getElementsByClassName),c.getById=ja(function(a){return o.appendChild(a).id=u,!n.getElementsByName||!n.getElementsByName(u).length}),c.getById?(d.filter.ID=function(a){var b=a.replace(_,aa);return function(a){return a.getAttribute("id")===b}},d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c?[c]:[]}}):(d.filter.ID=function(a){var b=a.replace(_,aa);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}},d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c,d,e,f=b.getElementById(a);if(f){if(c=f.getAttributeNode("id"),c&&c.value===a)return[f];e=b.getElementsByName(a),d=0;while(f=e[d++])if(c=f.getAttributeNode("id"),c&&c.value===a)return[f]}return[]}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElement
sByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){if("undefined"!=typeof b.getElementsByClassName&&p)return b.getElementsByClassName(a)},r=[],q=[],(c.qsa=Y.test(n.querySelectorAll))&&(ja(function(a){o.appendChild(a).innerHTML="<a id='"+u+"'></a><select id='"+u+"-\r\\' msallowcapture=''><option selected=''></option></select>",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+K+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+K+"*(?:value|"+J+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),ja(function(a){a.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var b=n.createElement("input");b.setAttri
bute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+K+"*[*^$|!~]?="),2!==a.querySelectorAll(":enabled").length&&q.push(":enabled",":disabled"),o.appendChild(a).disabled=!0,2!==a.querySelectorAll(":disabled").length&&q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=Y.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&ja(function(a){c.disconnectedMatch=s.call(a,"*"),s.call(a,"[s!='']:x"),r.push("!=",N)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=Y.test(o.compareDocumentPosition),t=b||Y.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===
b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===n||a.ownerDocument===v&&t(v,a)?-1:b===n||b.ownerDocument===v&&t(v,b)?1:k?I(k,a)-I(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,g=[a],h=[b];if(!e||!f)return a===n?-1:b===n?1:e?-1:f?1:k?I(k,a)-I(k,b):0;if(e===f)return la(a,b);c=a;while(c=c.parentNode)g.unshift(c);c=b;while(c=c.parentNode)h.unshift(c);while(g[d]===h[d])d++;return d?la(g[d],h[d]):g[d]===v?-1:h[d]===v?1:0},n):n},ga.matches=function(a,b){return ga(a,null,null,b)},ga.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(S,"='$1']"),c.matchesSelector&&p&&!A[b+" "]&&(!r||!r.test(b))&&(!q||!q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return ga(b,n,null,[a]).length>
0},ga.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},ga.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&C.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},ga.escape=function(a){return(a+"").replace(ba,ca)},ga.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},ga.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=ga.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=ga.selectors={cacheLength:50,createPseudo:ia,match:V,attrHandle:{},find:{},
relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(_,aa),a[3]=(a[3]||a[4]||a[5]||"").replace(_,aa),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||ga.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&ga.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return V.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&T.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(_,aa).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+K+")"+a+"("+K+"|$)"))&&y(a,function(a){return b.test("string"==typ
eof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=ga.attr(d,a);return null==e?"!="===b:!b||(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(O," ")+" ").indexOf(c)>-1:"|="===b&&(e===c||e.slice(0,c.length+1)===c+"-"))}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h,t=!1;if(q){if(f){while(p){m=b;while(m=m[p])if(h?m.nodeName.toLowerCase()===r:1===m.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){m=q,l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),j=k[a]||[],n=j[0]===w&&j[1],t=n&&j[2],m=n&&q.childNodes[n];while(m=+
+n&&m&&m[p]||(t=n=0)||o.pop())if(1===m.nodeType&&++t&&m===b){k[a]=[w,n,t];break}}else if(s&&(m=b,l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),j=k[a]||[],n=j[0]===w&&j[1],t=n),t===!1)while(m=++n&&m&&m[p]||(t=n=0)||o.pop())if((h?m.nodeName.toLowerCase()===r:1===m.nodeType)&&++t&&(s&&(l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),k[a]=[w,t]),m===b))break;return t-=e,t===d||t%d===0&&t/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||ga.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ia(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=I(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ia(function(a){var b=[],c=[],d=h(a.replace(P,"$1"));return d[u]?ia(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ia(function(a){return function(b
){return ga(a,b).length>0}}),contains:ia(function(a){return a=a.replace(_,aa),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ia(function(a){return U.test(a||"")||ga.error("unsupported lang: "+a),a=a.replace(_,aa).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:oa(!1),disabled:oa(!0),checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:funct
ion(a){return!d.pseudos.empty(a)},header:function(a){return X.test(a.nodeName)},input:function(a){return W.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:pa(function(){return[0]}),last:pa(function(a,b){return[b-1]}),eq:pa(function(a,b,c){return[c<0?c+b:c]}),even:pa(function(a,b){for(var c=0;c<b;c+=2)a.push(c);return a}),odd:pa(function(a,b){for(var c=1;c<b;c+=2)a.push(c);return a}),lt:pa(function(a,b,c){for(var d=c<0?c+b:c;--d>=0;)a.push(d);return a}),gt:pa(function(a,b,c){for(var d=c<0?c+b:c;++d<b;)a.push(d);return a})}},d.pseudos.nth=d.pseudos.eq;for(b in{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})d.pseudos[b]=ma(b);for(b in{submit:!0,reset:!0})d.pseudos[b]=na(b);function ra(){}ra.prototype=d.filters=d.pseudos,d.setFilters=new ra,g=ga.tokenize=function(a
,b){var c,e,f,g,h,i,j,k=z[a+" "];if(k)return b?0:k.slice(0);h=a,i=[],j=d.preFilter;while(h){c&&!(e=Q.exec(h))||(e&&(h=h.slice(e[0].length)||h),i.push(f=[])),c=!1,(e=R.exec(h))&&(c=e.shift(),f.push({value:c,type:e[0].replace(P," ")}),h=h.slice(c.length));for(g in d.filter)!(e=V[g].exec(h))||j[g]&&!(e=j[g](e))||(c=e.shift(),f.push({value:c,type:g,matches:e}),h=h.slice(c.length));if(!c)break}return b?h.length:h?ga.error(a):z(a,i).slice(0)};function sa(a){for(var b=0,c=a.length,d="";b<c;b++)d+=a[b].value;return d}function ta(a,b,c){var d=b.dir,e=b.next,f=e||d,g=c&&"parentNode"===f,h=x++;return b.first?function(b,c,e){while(b=b[d])if(1===b.nodeType||g)return a(b,c,e);return!1}:function(b,c,i){var j,k,l,m=[w,h];if(i){while(b=b[d])if((1===b.nodeType||g)&&a(b,c,i))return!0}else while(b=b[d])if(1===b.nodeType||g)if(l=b[u]||(b[u]={}),k=l[b.uniqueID]||(l[b.uniqueID]={}),e&&e===b.nodeName.toLowerCase())b=b[d]||b;else{if((j=k[f])&&j[0]===w&&j[1]===h)return m[2]=j[2];if(k[f]=m,m[2]=a(b,c,i))retur
n!0}return!1}}function ua(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function va(a,b,c){for(var d=0,e=b.length;d<e;d++)ga(a,b[d],c);return c}function wa(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;h<i;h++)(f=a[h])&&(c&&!c(f,d,e)||(g.push(f),j&&b.push(h)));return g}function xa(a,b,c,d,e,f){return d&&!d[u]&&(d=xa(d)),e&&!e[u]&&(e=xa(e,f)),ia(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||va(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:wa(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=wa(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?I(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=wa(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):G.apply(g,r)})}function ya(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=ta(function(a){return a===b},
h,!0),l=ta(function(a){return I(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];i<f;i++)if(c=d.relative[a[i].type])m=[ta(ua(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;e<f;e++)if(d.relative[a[e].type])break;return xa(i>1&&ua(m),i>1&&sa(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(P,"$1"),c,i<e&&ya(a.slice(i,e)),e<f&&ya(a=a.slice(e)),e<f&&sa(a))}m.push(c)}return ua(m)}function za(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,o,q,r=0,s="0",t=f&&[],u=[],v=j,x=f||e&&d.find.TAG("*",k),y=w+=null==v?1:Math.random()||.1,z=x.length;for(k&&(j=g===n||g||k);s!==z&&null!=(l=x[s]);s++){if(e&&l){o=0,g||l.ownerDocument===n||(m(l),h=!p);while(q=a[o++])if(q(l,g||n,h)){i.push(l);break}k&&(w=y)}c&&((l=!q&&l)&&r--,f&&t.push(l))}if(r+=s,c&&s!==r){o=0;while(q=b[o++])q(t,u,g,h);if(f){if(r>0)while(s--)t[s]||u[s]||(u[s]=E.call(i));u=wa(u)}G.apply(i,u),k&&!f&&u.length>0&&r+b.leng
th>1&&ga.uniqueSort(i)}return k&&(w=y,j=v),t};return c?ia(f):f}return h=ga.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=ya(b[c]),f[u]?d.push(f):e.push(f);f=A(a,za(e,d)),f.selector=a}return f},i=ga.select=function(a,b,c,e){var f,i,j,k,l,m="function"==typeof a&&a,n=!e&&g(a=m.selector||a);if(c=c||[],1===n.length){if(i=n[0]=n[0].slice(0),i.length>2&&"ID"===(j=i[0]).type&&9===b.nodeType&&p&&d.relative[i[1].type]){if(b=(d.find.ID(j.matches[0].replace(_,aa),b)||[])[0],!b)return c;m&&(b=b.parentNode),a=a.slice(i.shift().value.length)}f=V.needsContext.test(a)?0:i.length;while(f--){if(j=i[f],d.relative[k=j.type])break;if((l=d.find[k])&&(e=l(j.matches[0].replace(_,aa),$.test(i[0].type)&&qa(b.parentNode)||b))){if(i.splice(f,1),a=e.length&&sa(i),!a)return G.apply(c,e),c;break}}}return(m||h(a,n))(e,b,!p,c,!b||$.test(a)&&qa(b.parentNode)||b),c},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=ja(function(a){return
1&a.compareDocumentPosition(n.createElement("fieldset"))}),ja(function(a){return a.innerHTML="<a href='#'></a>","#"===a.firstChild.getAttribute("href")})||ka("type|href|height|width",function(a,b,c){if(!c)return a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&ja(function(a){return a.innerHTML="<input/>",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||ka("value",function(a,b,c){if(!c&&"input"===a.nodeName.toLowerCase())return a.defaultValue}),ja(function(a){return null==a.getAttribute("disabled")})||ka(J,function(a,b,c){var d;if(!c)return a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),ga}(a);r.find=x,r.expr=x.selectors,r.expr[":"]=r.expr.pseudos,r.uniqueSort=r.unique=x.uniqueSort,r.text=x.getText,r.isXMLDoc=x.isXML,r.contains=x.contains,r.escapeSelector=x.escape;var y=function(a,b,c){var d=[],e=void 0!==c;while((a=a[b])&&9!==a.nodeType)if(1===a.nodeType){if(e&&r(a).is(c))break;d.push(a)}return d},z=func
tion(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c},A=r.expr.match.needsContext;function B(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()}var C=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i,D=/^.[^:#\[\.,]*$/;function E(a,b,c){return r.isFunction(b)?r.grep(a,function(a,d){return!!b.call(a,d,a)!==c}):b.nodeType?r.grep(a,function(a){return a===b!==c}):"string"!=typeof b?r.grep(a,function(a){return i.call(b,a)>-1!==c}):D.test(b)?r.filter(b,a,c):(b=r.filter(b,a),r.grep(a,function(a){return i.call(b,a)>-1!==c&&1===a.nodeType}))}r.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?r.find.matchesSelector(d,a)?[d]:[]:r.find.matches(a,r.grep(b,function(a){return 1===a.nodeType}))},r.fn.extend({find:function(a){var b,c,d=this.length,e=this;if("string"!=typeof a)return this.pushStack(r(a).filter(function(){for(b=0;b<d;b++)if(r.contains(e[b],this))return!0}));for(c=this.pushStack([]),
b=0;b<d;b++)r.find(a,e[b],c);return d>1?r.uniqueSort(c):c},filter:function(a){return this.pushStack(E(this,a||[],!1))},not:function(a){return this.pushStack(E(this,a||[],!0))},is:function(a){return!!E(this,"string"==typeof a&&A.test(a)?r(a):a||[],!1).length}});var F,G=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/,H=r.fn.init=function(a,b,c){var e,f;if(!a)return this;if(c=c||F,"string"==typeof a){if(e="<"===a[0]&&">"===a[a.length-1]&&a.length>=3?[null,a,null]:G.exec(a),!e||!e[1]&&b)return!b||b.jquery?(b||c).find(a):this.constructor(b).find(a);if(e[1]){if(b=b instanceof r?b[0]:b,r.merge(this,r.parseHTML(e[1],b&&b.nodeType?b.ownerDocument||b:d,!0)),C.test(e[1])&&r.isPlainObject(b))for(e in b)r.isFunction(this[e])?this[e](b[e]):this.attr(e,b[e]);return this}return f=d.getElementById(e[2]),f&&(this[0]=f,this.length=1),this}return a.nodeType?(this[0]=a,this.length=1,this):r.isFunction(a)?void 0!==c.ready?c.ready(a):a(r):r.makeArray(a,this)};H.prototype=r.fn,F=r(d);var I=/^(?:parents|prev(?:Until|
All))/,J={children:!0,contents:!0,next:!0,prev:!0};r.fn.extend({has:function(a){var b=r(a,this),c=b.length;return this.filter(function(){for(var a=0;a<c;a++)if(r.contains(this,b[a]))return!0})},closest:function(a,b){var c,d=0,e=this.length,f=[],g="string"!=typeof a&&r(a);if(!A.test(a))for(;d<e;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&r.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?r.uniqueSort(f):f)},index:function(a){return a?"string"==typeof a?i.call(r(a),this[0]):i.call(this,a.jquery?a[0]:a):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(r.uniqueSort(r.merge(this.get(),r(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function K(a,b){while((a=a[b])&&1!==a.nodeType);return a}r.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return y(a,"parentNode
")},parentsUntil:function(a,b,c){return y(a,"parentNode",c)},next:function(a){return K(a,"nextSibling")},prev:function(a){return K(a,"previousSibling")},nextAll:function(a){return y(a,"nextSibling")},prevAll:function(a){return y(a,"previousSibling")},nextUntil:function(a,b,c){return y(a,"nextSibling",c)},prevUntil:function(a,b,c){return y(a,"previousSibling",c)},siblings:function(a){return z((a.parentNode||{}).firstChild,a)},children:function(a){return z(a.firstChild)},contents:function(a){return B(a,"iframe")?a.contentDocument:(B(a,"template")&&(a=a.content||a),r.merge([],a.childNodes))}},function(a,b){r.fn[a]=function(c,d){var e=r.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=r.filter(d,e)),this.length>1&&(J[a]||r.uniqueSort(e),I.test(a)&&e.reverse()),this.pushStack(e)}});var L=/[^\x20\t\r\n\f]+/g;function M(a){var b={};return r.each(a.match(L)||[],function(a,c){b[c]=!0}),b}r.Callbacks=function(a){a="string"==typeof a?M(a):r.extend({},a);var b,c,d,e,f=
[],g=[],h=-1,i=function(){for(e=e||a.once,d=b=!0;g.length;h=-1){c=g.shift();while(++h<f.length)f[h].apply(c[0],c[1])===!1&&a.stopOnFalse&&(h=f.length,c=!1)}a.memory||(c=!1),b=!1,e&&(f=c?[]:"")},j={add:function(){return f&&(c&&!b&&(h=f.length-1,g.push(c)),function d(b){r.each(b,function(b,c){r.isFunction(c)?a.unique&&j.has(c)||f.push(c):c&&c.length&&"string"!==r.type(c)&&d(c)})}(arguments),c&&!b&&i()),this},remove:function(){return r.each(arguments,function(a,b){var c;while((c=r.inArray(b,f,c))>-1)f.splice(c,1),c<=h&&h--}),this},has:function(a){return a?r.inArray(a,f)>-1:f.length>0},empty:function(){return f&&(f=[]),this},disable:function(){return e=g=[],f=c="",this},disabled:function(){return!f},lock:function(){return e=g=[],c||b||(f=c=""),this},locked:function(){return!!e},fireWith:function(a,c){return e||(c=c||[],c=[a,c.slice?c.slice():c],g.push(c),b||i()),this},fire:function(){return j.fireWith(this,arguments),this},fired:function(){return!!d}};return j};function N(a){return a}fu
nction O(a){throw a}function P(a,b,c,d){var e;try{a&&r.isFunction(e=a.promise)?e.call(a).done(b).fail(c):a&&r.isFunction(e=a.then)?e.call(a,b,c):b.apply(void 0,[a].slice(d))}catch(a){c.apply(void 0,[a])}}r.extend({Deferred:function(b){var c=[["notify","progress",r.Callbacks("memory"),r.Callbacks("memory"),2],["resolve","done",r.Callbacks("once memory"),r.Callbacks("once memory"),0,"resolved"],["reject","fail",r.Callbacks("once memory"),r.Callbacks("once memory"),1,"rejected"]],d="pending",e={state:function(){return d},always:function(){return f.done(arguments).fail(arguments),this},"catch":function(a){return e.then(null,a)},pipe:function(){var a=arguments;return r.Deferred(function(b){r.each(c,function(c,d){var e=r.isFunction(a[d[4]])&&a[d[4]];f[d[1]](function(){var a=e&&e.apply(this,arguments);a&&r.isFunction(a.promise)?a.promise().progress(b.notify).done(b.resolve).fail(b.reject):b[d[0]+"With"](this,e?[a]:arguments)})}),a=null}).promise()},then:function(b,d,e){var f=0;function g(b
,c,d,e){return function(){var h=this,i=arguments,j=function(){var a,j;if(!(b<f)){if(a=d.apply(h,i),a===c.promise())throw new TypeError("Thenable self-resolution");j=a&&("object"==typeof a||"function"==typeof a)&&a.then,r.isFunction(j)?e?j.call(a,g(f,c,N,e),g(f,c,O,e)):(f++,j.call(a,g(f,c,N,e),g(f,c,O,e),g(f,c,N,c.notifyWith))):(d!==N&&(h=void 0,i=[a]),(e||c.resolveWith)(h,i))}},k=e?j:function(){try{j()}catch(a){r.Deferred.exceptionHook&&r.Deferred.exceptionHook(a,k.stackTrace),b+1>=f&&(d!==O&&(h=void 0,i=[a]),c.rejectWith(h,i))}};b?k():(r.Deferred.getStackHook&&(k.stackTrace=r.Deferred.getStackHook()),a.setTimeout(k))}}return r.Deferred(function(a){c[0][3].add(g(0,a,r.isFunction(e)?e:N,a.notifyWith)),c[1][3].add(g(0,a,r.isFunction(b)?b:N)),c[2][3].add(g(0,a,r.isFunction(d)?d:O))}).promise()},promise:function(a){return null!=a?r.extend(a,e):e}},f={};return r.each(c,function(a,b){var g=b[2],h=b[5];e[b[1]]=g.add,h&&g.add(function(){d=h},c[3-a][2].disable,c[0][2].lock),g.add(b[3].fire),
f[b[0]]=function(){return f[b[0]+"With"](this===f?void 0:this,arguments),this},f[b[0]+"With"]=g.fireWith}),e.promise(f),b&&b.call(f,f),f},when:function(a){var b=arguments.length,c=b,d=Array(c),e=f.call(arguments),g=r.Deferred(),h=function(a){return function(c){d[a]=this,e[a]=arguments.length>1?f.call(arguments):c,--b||g.resolveWith(d,e)}};if(b<=1&&(P(a,g.done(h(c)).resolve,g.reject,!b),"pending"===g.state()||r.isFunction(e[c]&&e[c].then)))return g.then();while(c--)P(e[c],h(c),g.reject);return g.promise()}});var Q=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;r.Deferred.exceptionHook=function(b,c){a.console&&a.console.warn&&b&&Q.test(b.name)&&a.console.warn("jQuery.Deferred exception: "+b.message,b.stack,c)},r.readyException=function(b){a.setTimeout(function(){throw b})};var R=r.Deferred();r.fn.ready=function(a){return R.then(a)["catch"](function(a){r.readyException(a)}),this},r.extend({isReady:!1,readyWait:1,ready:function(a){(a===!0?--r.readyWait:r.isReady)||(r.isReady=!
0,a!==!0&&--r.readyWait>0||R.resolveWith(d,[r]))}}),r.ready.then=R.then;function S(){d.removeEventListener("DOMContentLoaded",S),
+a.removeEventListener("load",S),r.ready()}"complete"===d.readyState||"loading"!==d.readyState&&!d.documentElement.doScroll?a.setTimeout(r.ready):(d.addEventListener("DOMContentLoaded",S),a.addEventListener("load",S));var T=function(a,b,c,d,e,f,g){var h=0,i=a.length,j=null==c;if("object"===r.type(c)){e=!0;for(h in c)T(a,b,h,c[h],!0,f,g)}else if(void 0!==d&&(e=!0,r.isFunction(d)||(g=!0),j&&(g?(b.call(a,d),b=null):(j=b,b=function(a,b,c){return j.call(r(a),c)})),b))for(;h<i;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},U=function(a){return 1===a.nodeType||9===a.nodeType||!+a.nodeType};function V(){this.expando=r.expando+V.uid++}V.uid=1,V.prototype={cache:function(a){var b=a[this.expando];return b||(b={},U(a)&&(a.nodeType?a[this.expando]=b:Object.defineProperty(a,this.expando,{value:b,configurable:!0}))),b},set:function(a,b,c){var d,e=this.cache(a);if("string"==typeof b)e[r.camelCase(b)]=c;else for(d in b)e[r.camelCase(d)]=b[d];return e},get:function(a,
b){return void 0===b?this.cache(a):a[this.expando]&&a[this.expando][r.camelCase(b)]},access:function(a,b,c){return void 0===b||b&&"string"==typeof b&&void 0===c?this.get(a,b):(this.set(a,b,c),void 0!==c?c:b)},remove:function(a,b){var c,d=a[this.expando];if(void 0!==d){if(void 0!==b){Array.isArray(b)?b=b.map(r.camelCase):(b=r.camelCase(b),b=b in d?[b]:b.match(L)||[]),c=b.length;while(c--)delete d[b[c]]}(void 0===b||r.isEmptyObject(d))&&(a.nodeType?a[this.expando]=void 0:delete a[this.expando])}},hasData:function(a){var b=a[this.expando];return void 0!==b&&!r.isEmptyObject(b)}};var W=new V,X=new V,Y=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,Z=/[A-Z]/g;function $(a){return"true"===a||"false"!==a&&("null"===a?null:a===+a+""?+a:Y.test(a)?JSON.parse(a):a)}function _(a,b,c){var d;if(void 0===c&&1===a.nodeType)if(d="data-"+b.replace(Z,"-$&").toLowerCase(),c=a.getAttribute(d),"string"==typeof c){try{c=$(c)}catch(e){}X.set(a,b,c)}else c=void 0;return c}r.extend({hasData:function(a){return X.hasData(a)|
|W.hasData(a)},data:function(a,b,c){return X.access(a,b,c)},removeData:function(a,b){X.remove(a,b)},_data:function(a,b,c){return W.access(a,b,c)},_removeData:function(a,b){W.remove(a,b)}}),r.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=X.get(f),1===f.nodeType&&!W.get(f,"hasDataAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=r.camelCase(d.slice(5)),_(f,d,e[d])));W.set(f,"hasDataAttrs",!0)}return e}return"object"==typeof a?this.each(function(){X.set(this,a)}):T(this,function(b){var c;if(f&&void 0===b){if(c=X.get(f,a),void 0!==c)return c;if(c=_(f,a),void 0!==c)return c}else this.each(function(){X.set(this,a,b)})},null,b,arguments.length>1,null,!0)},removeData:function(a){return this.each(function(){X.remove(this,a)})}}),r.extend({queue:function(a,b,c){var d;if(a)return b=(b||"fx")+"queue",d=W.get(a,b),c&&(!d||Array.isArray(c)?d=W.access(a,b,r.makeArray(c)):d.push(c)),d||[]},dequeue:function(a,b){b=b||
"fx";var c=r.queue(a,b),d=c.length,e=c.shift(),f=r._queueHooks(a,b),g=function(){r.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return W.get(a,c)||W.access(a,c,{empty:r.Callbacks("once memory").add(function(){W.remove(a,[b+"queue",c])})})}}),r.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.length<c?r.queue(this[0],a):void 0===b?this:this.each(function(){var c=r.queue(this,a,b);r._queueHooks(this,a),"fx"===a&&"inprogress"!==c[0]&&r.dequeue(this,a)})},dequeue:function(a){return this.each(function(){r.dequeue(this,a)})},clearQueue:function(a){return this.queue(a||"fx",[])},promise:function(a,b){var c,d=1,e=r.Deferred(),f=this,g=this.length,h=function(){--d||e.resolveWith(f,[f])};"string"!=typeof a&&(b=a,a=void 0),a=a||"fx";while(g--)c=W.get(f[g],a+"queueHooks"),c&&c.empty&&(d++,c.empty.add(h));return h
(),e.promise(b)}});var aa=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,ba=new RegExp("^(?:([+-])=|)("+aa+")([a-z%]*)$","i"),ca=["Top","Right","Bottom","Left"],da=function(a,b){return a=b||a,"none"===a.style.display||""===a.style.display&&r.contains(a.ownerDocument,a)&&"none"===r.css(a,"display")},ea=function(a,b,c,d){var e,f,g={};for(f in b)g[f]=a.style[f],a.style[f]=b[f];e=c.apply(a,d||[]);for(f in b)a.style[f]=g[f];return e};function fa(a,b,c,d){var e,f=1,g=20,h=d?function(){return d.cur()}:function(){return r.css(a,b,"")},i=h(),j=c&&c[3]||(r.cssNumber[b]?"":"px"),k=(r.cssNumber[b]||"px"!==j&&+i)&&ba.exec(r.css(a,b));if(k&&k[3]!==j){j=j||k[3],c=c||[],k=+i||1;do f=f||".5",k/=f,r.style(a,b,k+j);while(f!==(f=h()/i)&&1!==f&&--g)}return c&&(k=+k||+i||0,e=c[1]?k+(c[1]+1)*c[2]:+c[2],d&&(d.unit=j,d.start=k,d.end=e)),e}var ga={};function ha(a){var b,c=a.ownerDocument,d=a.nodeName,e=ga[d];return e?e:(b=c.body.appendChild(c.createElement(d)),e=r.css(b,"display"),b.parentNode.removeChild(b),"
none"===e&&(e="block"),ga[d]=e,e)}function ia(a,b){for(var c,d,e=[],f=0,g=a.length;f<g;f++)d=a[f],d.style&&(c=d.style.display,b?("none"===c&&(e[f]=W.get(d,"display")||null,e[f]||(d.style.display="")),""===d.style.display&&da(d)&&(e[f]=ha(d))):"none"!==c&&(e[f]="none",W.set(d,"display",c)));for(f=0;f<g;f++)null!=e[f]&&(a[f].style.display=e[f]);return a}r.fn.extend({show:function(){return ia(this,!0)},hide:function(){return ia(this)},toggle:function(a){return"boolean"==typeof a?a?this.show():this.hide():this.each(function(){da(this)?r(this).show():r(this).hide()})}});var ja=/^(?:checkbox|radio)$/i,ka=/<([a-z][^\/\0>\x20\t\r\n\f]+)/i,la=/^$|\/(?:java|ecma)script/i,ma={option:[1,"<select multiple='multiple'>","</select>"],thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};ma.optgroup=ma.option,ma.tbody=ma.tfoot=ma.colgroup=ma.caption=ma.thead,m
a.th=ma.td;function na(a,b){var c;return c="undefined"!=typeof a.getElementsByTagName?a.getElementsByTagName(b||"*"):"undefined"!=typeof a.querySelectorAll?a.querySelectorAll(b||"*"):[],void 0===b||b&&B(a,b)?r.merge([a],c):c}function oa(a,b){for(var c=0,d=a.length;c<d;c++)W.set(a[c],"globalEval",!b||W.get(b[c],"globalEval"))}var pa=/<|&#?\w+;/;function qa(a,b,c,d,e){for(var f,g,h,i,j,k,l=b.createDocumentFragment(),m=[],n=0,o=a.length;n<o;n++)if(f=a[n],f||0===f)if("object"===r.type(f))r.merge(m,f.nodeType?[f]:f);else if(pa.test(f)){g=g||l.appendChild(b.createElement("div")),h=(ka.exec(f)||["",""])[1].toLowerCase(),i=ma[h]||ma._default,g.innerHTML=i[1]+r.htmlPrefilter(f)+i[2],k=i[0];while(k--)g=g.lastChild;r.merge(m,g.childNodes),g=l.firstChild,g.textContent=""}else m.push(b.createTextNode(f));l.textContent="",n=0;while(f=m[n++])if(d&&r.inArray(f,d)>-1)e&&e.push(f);else if(j=r.contains(f.ownerDocument,f),g=na(l.appendChild(f),"script"),j&&oa(g),c){k=0;while(f=g[k++])la.test(f.type||""
)&&c.push(f)}return l}!function(){var a=d.createDocumentFragment(),b=a.appendChild(d.createElement("div")),c=d.createElement("input");c.setAttribute("type","radio"),c.setAttribute("checked","checked"),c.setAttribute("name","t"),b.appendChild(c),o.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,b.innerHTML="<textarea>x</textarea>",o.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue}();var ra=d.documentElement,sa=/^key/,ta=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,ua=/^([^.]*)(?:\.(.+)|)/;function va(){return!0}function wa(){return!1}function xa(){try{return d.activeElement}catch(a){}}function ya(a,b,c,d,e,f){var g,h;if("object"==typeof b){"string"!=typeof c&&(d=d||c,c=void 0);for(h in b)ya(a,h,c,d,b[h],f);return a}if(null==d&&null==e?(e=c,d=c=void 0):null==e&&("string"==typeof c?(e=d,d=void 0):(e=d,d=c,c=void 0)),e===!1)e=wa;else if(!e)return a;return 1===f&&(g=e,e=function(a){return r().off(a),g.apply(this,arguments)},e.guid=g.guid||(g.guid=r.guid++)),a.each(
function(){r.event.add(this,b,e,d,c)})}r.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,n,o,p,q=W.get(a);if(q){c.handler&&(f=c,c=f.handler,e=f.selector),e&&r.find.matchesSelector(ra,e),c.guid||(c.guid=r.guid++),(i=q.events)||(i=q.events={}),(g=q.handle)||(g=q.handle=function(b){return"undefined"!=typeof r&&r.event.triggered!==b.type?r.event.dispatch.apply(a,arguments):void 0}),b=(b||"").match(L)||[""],j=b.length;while(j--)h=ua.exec(b[j])||[],n=p=h[1],o=(h[2]||"").split(".").sort(),n&&(l=r.event.special[n]||{},n=(e?l.delegateType:l.bindType)||n,l=r.event.special[n]||{},k=r.extend({type:n,origType:p,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&r.expr.match.needsContext.test(e),namespace:o.join(".")},f),(m=i[n])||(m=i[n]=[],m.delegateCount=0,l.setup&&l.setup.call(a,d,o,g)!==!1||a.addEventListener&&a.addEventListener(n,g)),l.add&&(l.add.call(a,k),k.handler.guid||(k.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,k):m.push(k),r.event.global[n]=!0)}},remove:f
unction(a,b,c,d,e){var f,g,h,i,j,k,l,m,n,o,p,q=W.hasData(a)&&W.get(a);if(q&&(i=q.events)){b=(b||"").match(L)||[""],j=b.length;while(j--)if(h=ua.exec(b[j])||[],n=p=h[1],o=(h[2]||"").split(".").sort(),n){l=r.event.special[n]||{},n=(d?l.delegateType:l.bindType)||n,m=i[n]||[],h=h[2]&&new RegExp("(^|\\.)"+o.join("\\.(?:.*\\.|)")+"(\\.|$)"),g=f=m.length;while(f--)k=m[f],!e&&p!==k.origType||c&&c.guid!==k.guid||h&&!h.test(k.namespace)||d&&d!==k.selector&&("**"!==d||!k.selector)||(m.splice(f,1),k.selector&&m.delegateCount--,l.remove&&l.remove.call(a,k));g&&!m.length&&(l.teardown&&l.teardown.call(a,o,q.handle)!==!1||r.removeEvent(a,n,q.handle),delete i[n])}else for(n in i)r.event.remove(a,n+b[j],c,d,!0);r.isEmptyObject(i)&&W.remove(a,"handle events")}},dispatch:function(a){var b=r.event.fix(a),c,d,e,f,g,h,i=new Array(arguments.length),j=(W.get(this,"events")||{})[b.type]||[],k=r.event.special[b.type]||{};for(i[0]=b,c=1;c<arguments.length;c++)i[c]=arguments[c];if(b.delegateTarget=this,!k.preDi
spatch||k.preDispatch.call(this,b)!==!1){h=r.event.handlers.call(this,b,j),c=0;while((f=h[c++])&&!b.isPropagationStopped()){b.currentTarget=f.elem,d=0;while((g=f.handlers[d++])&&!b.isImmediatePropagationStopped())b.rnamespace&&!b.rnamespace.test(g.namespace)||(b.handleObj=g,b.data=g.data,e=((r.event.special[g.origType]||{}).handle||g.handler).apply(f.elem,i),void 0!==e&&(b.result=e)===!1&&(b.preventDefault(),b.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,b),b.result}},handlers:function(a,b){var c,d,e,f,g,h=[],i=b.delegateCount,j=a.target;if(i&&j.nodeType&&!("click"===a.type&&a.button>=1))for(;j!==this;j=j.parentNode||this)if(1===j.nodeType&&("click"!==a.type||j.disabled!==!0)){for(f=[],g={},c=0;c<i;c++)d=b[c],e=d.selector+" ",void 0===g[e]&&(g[e]=d.needsContext?r(e,this).index(j)>-1:r.find(e,this,null,[j]).length),g[e]&&f.push(d);f.length&&h.push({elem:j,handlers:f})}return j=this,i<b.length&&h.push({elem:j,handlers:b.slice(i)}),h},addProp:function(a,b){Object
.defineProperty(r.Event.prototype,a,{enumerable:!0,configurable:!0,get:r.isFunction(b)?function(){if(this.originalEvent)return b(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[a]},set:function(b){Object.defineProperty(this,a,{enumerable:!0,configurable:!0,writable:!0,value:b})}})},fix:function(a){return a[r.expando]?a:new r.Event(a)},special:{load:{noBubble:!0},focus:{trigger:function(){if(this!==xa()&&this.focus)return this.focus(),!1},delegateType:"focusin"},blur:{trigger:function(){if(this===xa()&&this.blur)return this.blur(),!1},delegateType:"focusout"},click:{trigger:function(){if("checkbox"===this.type&&this.click&&B(this,"input"))return this.click(),!1},_default:function(a){return B(a.target,"a")}},beforeunload:{postDispatch:function(a){void 0!==a.result&&a.originalEvent&&(a.originalEvent.returnValue=a.result)}}}},r.removeEvent=function(a,b,c){a.removeEventListener&&a.removeEventListener(b,c)},r.Event=function(a,b){return this instanceof r.Eve
nt?(a&&a.type?(this.originalEvent=a,this.type=a.type,this.isDefaultPrevented=a.defaultPrevented||void 0===a.defaultPrevented&&a.returnValue===!1?va:wa,this.target=a.target&&3===a.target.nodeType?a.target.parentNode:a.target,this.currentTarget=a.currentTarget,this.relatedTarget=a.relatedTarget):this.type=a,b&&r.extend(this,b),this.timeStamp=a&&a.timeStamp||r.now(),void(this[r.expando]=!0)):new r.Event(a,b)},r.Event.prototype={constructor:r.Event,isDefaultPrevented:wa,isPropagationStopped:wa,isImmediatePropagationStopped:wa,isSimulated:!1,preventDefault:function(){var a=this.originalEvent;this.isDefaultPrevented=va,a&&!this.isSimulated&&a.preventDefault()},stopPropagation:function(){var a=this.originalEvent;this.isPropagationStopped=va,a&&!this.isSimulated&&a.stopPropagation()},stopImmediatePropagation:function(){var a=this.originalEvent;this.isImmediatePropagationStopped=va,a&&!this.isSimulated&&a.stopImmediatePropagation(),this.stopPropagation()}},r.each({altKey:!0,bubbles:!0,cancel
able:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:function(a){var b=a.button;return null==a.which&&sa.test(a.type)?null!=a.charCode?a.charCode:a.keyCode:!a.which&&void 0!==b&&ta.test(a.type)?1&b?1:2&b?3:4&b?2:0:a.which}},r.event.addProp),r.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(a,b){r.event.special[a]={delegateType:b,bindType:b,handle:function(a){var c,d=this,e=a.relatedTarget,f=a.handleObj;return e&&(e===d||r.contains(d,e))||(a.type=f.origType,c=f.handler.apply(this,arguments),a.type=b),c}}}),r.fn.extend({on:function(a,b,c,d){return ya(this,a,b,c,d)},one:function(a,b,c,d){return ya(this,a,b,c,d,1)},off:function(a,b,c){var d,e;if(a&&a.preventDefault&&a
.handleObj)return d=a.handleObj,r(a.delegateTarget).off(d.namespace?d.origType+"."+d.namespace:d.origType,d.selector,d.handler),this;if("object"==typeof a){for(e in a)this.off(e,b,a[e]);return this}return b!==!1&&"function"!=typeof b||(c=b,b=void 0),c===!1&&(c=wa),this.each(function(){r.event.remove(this,a,c,b)})}});var za=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([a-z][^\/\0>\x20\t\r\n\f]*)[^>]*)\/>/gi,Aa=/<script|<style|<link/i,Ba=/checked\s*(?:[^=]|=\s*.checked.)/i,Ca=/^true\/(.*)/,Da=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function Ea(a,b){return B(a,"table")&&B(11!==b.nodeType?b:b.firstChild,"tr")?r(">tbody",a)[0]||a:a}function Fa(a){return a.type=(null!==a.getAttribute("type"))+"/"+a.type,a}function Ga(a){var b=Ca.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function Ha(a,b){var c,d,e,f,g,h,i,j;if(1===b.nodeType){if(W.hasData(a)&&(f=W.access(a),g=W.set(b,f),j=f.events)){delete g.handle,g.events={};for(e in j)for(c=0,d=j[e].length;c<d;c++)r.event
.add(b,e,j[e][c])}X.hasData(a)&&(h=X.access(a),i=r.extend({},h),X.set(b,i))}}function Ia(a,b){var c=b.nodeName.toLowerCase();"input"===c&&ja.test(a.type)?b.checked=a.checked:"input"!==c&&"textarea"!==c||(b.defaultValue=a.defaultValue)}function Ja(a,b,c,d){b=g.apply([],b);var e,f,h,i,j,k,l=0,m=a.length,n=m-1,q=b[0],s=r.isFunction(q);if(s||m>1&&"string"==typeof q&&!o.checkClone&&Ba.test(q))return a.each(function(e){var f=a.eq(e);s&&(b[0]=q.call(this,e,f.html())),Ja(f,b,c,d)});if(m&&(e=qa(b,a[0].ownerDocument,!1,a,d),f=e.firstChild,1===e.childNodes.length&&(e=f),f||d)){for(h=r.map(na(e,"script"),Fa),i=h.length;l<m;l++)j=e,l!==n&&(j=r.clone(j,!0,!0),i&&r.merge(h,na(j,"script"))),c.call(a[l],j,l);if(i)for(k=h[h.length-1].ownerDocument,r.map(h,Ga),l=0;l<i;l++)j=h[l],la.test(j.type||"")&&!W.access(j,"globalEval")&&r.contains(k,j)&&(j.src?r._evalUrl&&r._evalUrl(j.src):p(j.textContent.replace(Da,""),k))}return a}function Ka(a,b,c){for(var d,e=b?r.filter(b,a):a,f=0;null!=(d=e[f]);f++)c||1!==d
.nodeType||r.cleanData(na(d)),d.parentNode&&(c&&r.contains(d.ownerDocument,d)&&oa(na(d,"script")),d.parentNode.removeChild(d));return a}r.extend({htmlPrefilter:function(a){return a.replace(za,"<$1></$2>")},clone:function(a,b,c){var d,e,f,g,h=a.cloneNode(!0),i=r.contains(a.ownerDocument,a);if(!(o.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||r.isXMLDoc(a)))for(g=na(h),f=na(a),d=0,e=f.length;d<e;d++)Ia(f[d],g[d]);if(b)if(c)for(f=f||na(a),g=g||na(h),d=0,e=f.length;d<e;d++)Ha(f[d],g[d]);else Ha(a,h);return g=na(h,"script"),g.length>0&&oa(g,!i&&na(a,"script")),h},cleanData:function(a){for(var b,c,d,e=r.event.special,f=0;void 0!==(c=a[f]);f++)if(U(c)){if(b=c[W.expando]){if(b.events)for(d in b.events)e[d]?r.event.remove(c,d):r.removeEvent(c,d,b.handle);c[W.expando]=void 0}c[X.expando]&&(c[X.expando]=void 0)}}}),r.fn.extend({detach:function(a){return Ka(this,a,!0)},remove:function(a){return Ka(this,a)},text:function(a){return T(this,function(a){return void 0===a?r.text(this):this.empty()
.each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=a)})},null,a,arguments.length)},append:function(){return Ja(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Ea(this,a);b.appendChild(a)}})},prepend:function(){return Ja(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Ea(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return Ja(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return Ja(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},empty:function(){for(var a,b=0;null!=(a=this[b]);b++)1===a.nodeType&&(r.cleanData(na(a,!1)),a.textContent="");return this},clone:function(a,b){return a=null!=a&&a,b=null==b?a:b,this.map(function(){return r.clone(this,a,b)})},html:function(a){return T(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0=
==a&&1===b.nodeType)return b.innerHTML;if("string"==typeof a&&!Aa.test(a)&&!ma[(ka.exec(a)||["",""])[1].toLowerCase()]){a=r.htmlPrefilter(a);try{for(;c<d;c++)b=this[c]||{},1===b.nodeType&&(r.cleanData(na(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=[];return Ja(this,arguments,function(b){var c=this.parentNode;r.inArray(this,a)<0&&(r.cleanData(na(this)),c&&c.replaceChild(b,this))},a)}}),r.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){r.fn[a]=function(a){for(var c,d=[],e=r(a),f=e.length-1,g=0;g<=f;g++)c=g===f?this:this.clone(!0),r(e[g])[b](c),h.apply(d,c.get());return this.pushStack(d)}});var La=/^margin/,Ma=new RegExp("^("+aa+")(?!px)[a-z%]+$","i"),Na=function(b){var c=b.ownerDocument.defaultView;return c&&c.opener||(c=a),c.getComputedStyle(b)};!function(){function b(){if(i){i.style.cssText="box-sizing:border-box;position:relative;d
isplay:block;margin:auto;border:1px;padding:1px;top:1%;width:50%",i.innerHTML="",ra.appendChild(h);var b=a.getComputedStyle(i);c="1%"!==b.top,g="2px"===b.marginLeft,e="4px"===b.width,i.style.marginRight="50%",f="4px"===b.marginRight,ra.removeChild(h),i=null}}var c,e,f,g,h=d.createElement("div"),i=d.createElement("div");i.style&&(i.style.backgroundClip="content-box",i.cloneNode(!0).style.backgroundClip="",o.clearCloneStyle="content-box"===i.style.backgroundClip,h.style.cssText="border:0;width:8px;height:0;top:0;left:-9999px;padding:0;margin-top:1px;position:absolute",h.appendChild(i),r.extend(o,{pixelPosition:function(){return b(),c},boxSizingReliable:function(){return b(),e},pixelMarginRight:function(){return b(),f},reliableMarginLeft:function(){return b(),g}}))}();function Oa(a,b,c){var d,e,f,g,h=a.style;return c=c||Na(a),c&&(g=c.getPropertyValue(b)||c[b],""!==g||r.contains(a.ownerDocument,a)||(g=r.style(a,b)),!o.pixelMarginRight()&&Ma.test(g)&&La.test(b)&&(d=h.width,e=h.minWidth,f
=h.maxWidth,h.minWidth=h.maxWidth=h.width=g,g=c.width,h.width=d,h.minWidth=e,h.maxWidth=f)),void 0!==g?g+"":g}function Pa(a,b){return{get:function(){return a()?void delete this.get:(this.get=b).apply(this,arguments)}}}var Qa=/^(none|table(?!-c[ea]).+)/,Ra=/^--/,Sa={position:"absolute",visibility:"hidden",display:"block"},Ta={letterSpacing:"0",fontWeight:"400"},Ua=["Webkit","Moz","ms"],Va=d.createElement("div").style;function Wa(a){if(a in Va)return a;var b=a[0].toUpperCase()+a.slice(1),c=Ua.length;while(c--)if(a=Ua[c]+b,a in Va)return a}function Xa(a){var b=r.cssProps[a];return b||(b=r.cssProps[a]=Wa(a)||a),b}function Ya(a,b,c){var d=ba.exec(b);return d?Math.max(0,d[2]-(c||0))+(d[3]||"px"):b}function Za(a,b,c,d,e){var f,g=0;for(f=c===(d?"border":"content")?4:"width"===b?1:0;f<4;f+=2)"margin"===c&&(g+=r.css(a,c+ca[f],!0,e)),d?("content"===c&&(g-=r.css(a,"padding"+ca[f],!0,e)),"margin"!==c&&(g-=r.css(a,"border"+ca[f]+"Width",!0,e))):(g+=r.css(a,"padding"+ca[f],!0,e),"padding"!==c&&(g+
=r.css(a,"border"+ca[f]+"Width",!0,e)));return g}function $a(a,b,c){var d,e=Na(a),f=Oa(a,b,e),g="border-box"===r.css(a,"boxSizing",!1,e);return Ma.test(f)?f:(d=g&&(o.boxSizingReliable()||f===a.style[b]),"auto"===f&&(f=a["offset"+b[0].toUpperCase()+b.slice(1)]),f=parseFloat(f)||0,f+Za(a,b,c||(g?"border":"content"),d,e)+"px")}r.extend({cssHooks:{opacity:{get:function(a,b){if(b){var c=Oa(a,"opacity");return""===c?"1":c}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":"cssFloat"},style:function(a,b,c,d){if(a&&3!==a.nodeType&&8!==a.nodeType&&a.style){var e,f,g,h=r.camelCase(b),i=Ra.test(b),j=a.style;return i||(b=Xa(h)),g=r.cssHooks[b]||r.cssHooks[h],void 0===c?g&&"get"in g&&void 0!==(e=g.get(a,!1,d))?e:j[b]:(f=typeof c,"string"===f&&(e=ba.exec(c))&&e[1]&&(c=fa(a,b,e),f="number"),null!=c&&c===c&&("number"===f&&(c+=e&&e[3]||(r.cssNumber[
h]?"":"px")),o.clearCloneStyle||""!==c||0!==b.indexOf("background")||(j[b]="inherit"),g&&"set"in g&&void 0===(c=g.set(a,c,d))||(i?j.setProperty(b,c):j[b]=c)),void 0)}},css:function(a,b,c,d){var e,f,g,h=r.camelCase(b),i=Ra.test(b);return i||(b=Xa(h)),g=r.cssHooks[b]||r.cssHooks[h],g&&"get"in g&&(e=g.get(a,!0,c)),void 0===e&&(e=Oa(a,b,d)),"normal"===e&&b in Ta&&(e=Ta[b]),""===c||c?(f=parseFloat(e),c===!0||isFinite(f)?f||0:e):e}}),r.each(["height","width"],function(a,b){r.cssHooks[b]={get:function(a,c,d){if(c)return!Qa.test(r.css(a,"display"))||a.getClientRects().length&&a.getBoundingClientRect().width?$a(a,b,d):ea(a,Sa,function(){return $a(a,b,d)})},set:function(a,c,d){var e,f=d&&Na(a),g=d&&Za(a,b,d,"border-box"===r.css(a,"boxSizing",!1,f),f);return g&&(e=ba.exec(c))&&"px"!==(e[3]||"px")&&(a.style[b]=c,c=r.css(a,b)),Ya(a,c,g)}}}),r.cssHooks.marginLeft=Pa(o.reliableMarginLeft,function(a,b){if(b)return(parseFloat(Oa(a,"marginLeft"))||a.getBoundingClientRect().left-ea(a,{marginLeft:0},fu
nction(){return a.getBoundingClientRect().left}))+"px"}),r.each({margin:"",padding:"",border:"Width"},function(a,b){r.cssHooks[a+b]={expand:function(c){for(var d=0,e={},f="string"==typeof c?c.split(" "):[c];d<4;d++)e[a+ca[d]+b]=f[d]||f[d-2]||f[0];return e}},La.test(a)||(r.cssHooks[a+b].set=Ya)}),r.fn.extend({css:function(a,b){return T(this,function(a,b,c){var d,e,f={},g=0;if(Array.isArray(b)){for(d=Na(a),e=b.length;g<e;g++)f[b[g]]=r.css(a,b[g],!1,d);return f}return void 0!==c?r.style(a,b,c):r.css(a,b)},a,b,arguments.length>1)}});function _a(a,b,c,d,e){return new _a.prototype.init(a,b,c,d,e)}r.Tween=_a,_a.prototype={constructor:_a,init:function(a,b,c,d,e,f){this.elem=a,this.prop=c,this.easing=e||r.easing._default,this.options=b,this.start=this.now=this.cur(),this.end=d,this.unit=f||(r.cssNumber[c]?"":"px")},cur:function(){var a=_a.propHooks[this.prop];return a&&a.get?a.get(this):_a.propHooks._default.get(this)},run:function(a){var b,c=_a.propHooks[this.prop];return this.options.durat
ion?this.pos=b=r.easing[this.easing](a,this.options.duration*a,0,1,this.options.duration):this.pos=b=a,this.now=(this.end-this.start)*b+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),c&&c.set?c.set(this):_a.propHooks._default.set(this),this}},_a.prototype.init.prototype=_a.prototype,_a.propHooks={_default:{get:function(a){var b;return 1!==a.elem.nodeType||null!=a.elem[a.prop]&&null==a.elem.style[a.prop]?a.elem[a.prop]:(b=r.css(a.elem,a.prop,""),b&&"auto"!==b?b:0)},set:function(a){r.fx.step[a.prop]?r.fx.step[a.prop](a):1!==a.elem.nodeType||null==a.elem.style[r.cssProps[a.prop]]&&!r.cssHooks[a.prop]?a.elem[a.prop]=a.now:r.style(a.elem,a.prop,a.now+a.unit)}}},_a.propHooks.scrollTop=_a.propHooks.scrollLeft={set:function(a){a.elem.nodeType&&a.elem.parentNode&&(a.elem[a.prop]=a.now)}},r.easing={linear:function(a){return a},swing:function(a){return.5-Math.cos(a*Math.PI)/2},_default:"swing"},r.fx=_a.prototype.init,r.fx.step={};var ab,bb,cb=/^(?:toggle|show|hid
e)$/,db=/queueHooks$/;function eb(){bb&&(d.hidden===!1&&a.requestAnimationFrame?a.requestAnimationFrame(eb):a.setTimeout(eb,r.fx.interval),r.fx.tick())}function fb(){return a.setTimeout(function(){ab=void 0}),ab=r.now()}function gb(a,b){var c,d=0,e={height:a};for(b=b?1:0;d<4;d+=2-b)c=ca[d],e["margin"+c]=e["padding"+c]=a;return b&&(e.opacity=e.width=a),e}function hb(a,b,c){for(var d,e=(kb.tweeners[b]||[]).concat(kb.tweeners["*"]),f=0,g=e.length;f<g;f++)if(d=e[f].call(c,b,a))return d}function ib(a,b,c){var d,e,f,g,h,i,j,k,l="width"in b||"height"in b,m=this,n={},o=a.style,p=a.nodeType&&da(a),q=W.get(a,"fxshow");c.queue||(g=r._queueHooks(a,"fx"),null==g.unqueued&&(g.unqueued=0,h=g.empty.fire,g.empty.fire=function(){g.unqueued||h()}),g.unqueued++,m.always(function(){m.always(function(){g.unqueued--,r.queue(a,"fx").length||g.empty.fire()})}));for(d in b)if(e=b[d],cb.test(e)){if(delete b[d],f=f||"toggle"===e,e===(p?"hide":"show")){if("show"!==e||!q||void 0===q[d])continue;p=!0}n[d]=q&&q[d]
||r.style(a,d)}if(i=!r.isEmptyObject(b),i||!r.isEmptyObject(n)){l&&1===a.nodeType&&(c.overflow=[o.overflow,o.overflowX,o.overflowY],j=q&&q.display,null==j&&(j=W.get(a,"display")),k=r.css(a,"display"),"none"===k&&(j?k=j:(ia([a],!0),j=a.style.display||j,k=r.css(a,"display"),ia([a]))),("inline"===k||"inline-block"===k&&null!=j)&&"none"===r.css(a,"float")&&(i||(m.done(function(){o.display=j}),null==j&&(k=o.display,j="none"===k?"":k)),o.display="inline-block")),c.overflow&&(o.overflow="hidden",m.always(function(){o.overflow=c.overflow[0],o.overflowX=c.overflow[1],o.overflowY=c.overflow[2]})),i=!1;for(d in n)i||(q?"hidden"in q&&(p=q.hidden):q=W.access(a,"fxshow",{display:j}),f&&(q.hidden=!p),p&&ia([a],!0),m.done(function(){p||ia([a]),W.remove(a,"fxshow");for(d in n)r.style(a,d,n[d])})),i=hb(p?q[d]:0,d,m),d in q||(q[d]=i.start,p&&(i.end=i.start,i.start=0))}}function jb(a,b){var c,d,e,f,g;for(c in a)if(d=r.camelCase(c),e=b[d],f=a[c],Array.isArray(f)&&(e=f[1],f=a[c]=f[0]),c!==d&&(a[d]=f,dele
te a[c]),g=r.cssHooks[d],g&&"expand"in g){f=g.expand(f),delete a[d];for(c in f)c in a||(a[c]=f[c],b[c]=e)}else b[d]=e}function kb(a,b,c){var d,e,f=0,g=kb.prefilters.length,h=r.Deferred().always(function(){delete i.elem}),i=function(){if(e)return!1;for(var b=ab||fb(),c=Math.max(0,j.startTime+j.duration-b),d=c/j.duration||0,f=1-d,g=0,i=j.tweens.length;g<i;g++)j.tweens[g].run(f);return h.notifyWith(a,[j,f,c]),f<1&&i?c:(i||h.notifyWith(a,[j,1,0]),h.resolveWith(a,[j]),!1)},j=h.promise({elem:a,props:r.extend({},b),opts:r.extend(!0,{specialEasing:{},easing:r.easing._default},c),originalProperties:b,originalOptions:c,startTime:ab||fb(),duration:c.duration,tweens:[],createTween:function(b,c){var d=r.Tween(a,j.opts,b,c,j.opts.specialEasing[b]||j.opts.easing);return j.tweens.push(d),d},stop:function(b){var c=0,d=b?j.tweens.length:0;if(e)return this;for(e=!0;c<d;c++)j.tweens[c].run(1);return b?(h.notifyWith(a,[j,1,0]),h.resolveWith(a,[j,b])):h.rejectWith(a,[j,b]),this}}),k=j.props;for(jb(k,j.op
ts.specialEasing);f<g;f++)if(d=kb.prefilters[f].call(j,a,k,j.opts))return r.isFunction(d.stop)&&(r._queueHooks(j.elem,j.opts.queue).stop=r.proxy(d.stop,d)),d;return r.map(k,hb,j),r.isFunction(j.opts.start)&&j.opts.start.call(a,j),j.progress(j.opts.progress).done(j.opts.done,j.opts.complete).fail(j.opts.fail).always(j.opts.always),r.fx.timer(r.extend(i,{elem:a,anim:j,queue:j.opts.queue})),j}r.Animation=r.extend(kb,{tweeners:{"*":[function(a,b){var c=this.createTween(a,b);return fa(c.elem,a,ba.exec(b),c),c}]},tweener:function(a,b){r.isFunction(a)?(b=a,a=["*"]):a=a.match(L);for(var c,d=0,e=a.length;d<e;d++)c=a[d],kb.tweeners[c]=kb.tweeners[c]||[],kb.tweeners[c].unshift(b)},prefilters:[ib],prefilter:function(a,b){b?kb.prefilters.unshift(a):kb.prefilters.push(a)}}),r.speed=function(a,b,c){var d=a&&"object"==typeof a?r.extend({},a):{complete:c||!c&&b||r.isFunction(a)&&a,duration:a,easing:c&&b||b&&!r.isFunction(b)&&b};return r.fx.off?d.duration=0:"number"!=typeof d.duration&&(d.duration in
r.fx.speeds?d.duration=r.fx.speeds[d.duration]:d.duration=r.fx.speeds._default),null!=d.queue&&d.queue!==!0||(d.queue="fx"),d.old=d.complete,d.complete=function(){r.isFunction(d.old)&&d.old.call(this),d.queue&&r.dequeue(this,d.queue)},d},r.fn.extend({fadeTo:function(a,b,c,d){return this.filter(da).css("opacity",0).show().end().animate({opacity:b},a,c,d)},animate:function(a,b,c,d){var e=r.isEmptyObject(a),f=r.speed(b,c,d),g=function(){var b=kb(this,r.extend({},a),f);(e||W.get(this,"finish"))&&b.stop(!0)};return g.finish=g,e||f.queue===!1?this.each(g):this.queue(f.queue,g)},stop:function(a,b,c){var d=function(a){var b=a.stop;delete a.stop,b(c)};return"string"!=typeof a&&(c=b,b=a,a=void 0),b&&a!==!1&&this.queue(a||"fx",[]),this.each(function(){var b=!0,e=null!=a&&a+"queueHooks",f=r.timers,g=W.get(this);if(e)g[e]&&g[e].stop&&d(g[e]);else for(e in g)g[e]&&g[e].stop&&db.test(e)&&d(g[e]);for(e=f.length;e--;)f[e].elem!==this||null!=a&&f[e].queue!==a||(f[e].anim.stop(c),b=!1,f.splice(e,1));
!b&&c||r.dequeue(this,a)})},finish:function(a){return a!==!1&&(a=a||"fx"),this.each(function(){var b,c=W.get(this),d=c[a+"queue"],e=c[a+"queueHooks"],f=r.timers,g=d?d.length:0;for(c.finish=!0,r.queue(this,a,[]),e&&e.stop&&e.stop.call(this,!0),b=f.length;b--;)f[b].elem===this&&f[b].queue===a&&(f[b].anim.stop(!0),f.splice(b,1));for(b=0;b<g;b++)d[b]&&d[b].finish&&d[b].finish.call(this);delete c.finish})}}),r.each(["toggle","show","hide"],function(a,b){var c=r.fn[b];r.fn[b]=function(a,d,e){return null==a||"boolean"==typeof a?c.apply(this,arguments):this.animate(gb(b,!0),a,d,e)}}),r.each({slideDown:gb("show"),slideUp:gb("hide"),slideToggle:gb("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(a,b){r.fn[a]=function(a,c,d){return this.animate(b,a,c,d)}}),r.timers=[],r.fx.tick=function(){var a,b=0,c=r.timers;for(ab=r.now();b<c.length;b++)a=c[b],a()||c[b]!==a||c.splice(b--,1);c.length||r.fx.stop(),ab=void 0},r.fx.timer=function(a){r.timers.push
(a),r.fx.start()},r.fx.interval=13,r.fx.start=function(){bb||(bb=!0,eb())},r.fx.stop=function(){bb=null},r.fx.speeds={slow:600,fast:200,_default:400},r.fn.delay=function(b,c){return b=r.fx?r.fx.speeds[b]||b:b,c=c||"fx",this.queue(c,function(c,d){var e=a.setTimeout(c,b);d.stop=function(){a.clearTimeout(e)}})},function(){var a=d.createElement("input"),b=d.createElement("select"),c=b.appendChild(d.createElement("option"));a.type="checkbox",o.checkOn=""!==a.value,o.optSelected=c.selected,a=d.createElement("input"),a.value="t",a.type="radio",o.radioValue="t"===a.value}();var lb,mb=r.expr.attrHandle;r.fn.extend({attr:function(a,b){return T(this,r.attr,a,b,arguments.length>1)},removeAttr:function(a){return this.each(function(){r.removeAttr(this,a)})}}),r.extend({attr:function(a,b,c){var d,e,f=a.nodeType;if(3!==f&&8!==f&&2!==f)return"undefined"==typeof a.getAttribute?r.prop(a,b,c):(1===f&&r.isXMLDoc(a)||(e=r.attrHooks[b.toLowerCase()]||(r.expr.match.bool.test(b)?lb:void 0)),void 0!==c?null=
==c?void r.removeAttr(a,b):e&&"set"in e&&void 0!==(d=e.set(a,c,b))?d:(a.setAttribute(b,c+""),c):e&&"get"in e&&null!==(d=e.get(a,b))?d:(d=r.find.attr(a,b),
+null==d?void 0:d))},attrHooks:{type:{set:function(a,b){if(!o.radioValue&&"radio"===b&&B(a,"input")){var c=a.value;return a.setAttribute("type",b),c&&(a.value=c),b}}}},removeAttr:function(a,b){var c,d=0,e=b&&b.match(L);if(e&&1===a.nodeType)while(c=e[d++])a.removeAttribute(c)}}),lb={set:function(a,b,c){return b===!1?r.removeAttr(a,c):a.setAttribute(c,c),c}},r.each(r.expr.match.bool.source.match(/\w+/g),function(a,b){var c=mb[b]||r.find.attr;mb[b]=function(a,b,d){var e,f,g=b.toLowerCase();return d||(f=mb[g],mb[g]=e,e=null!=c(a,b,d)?g:null,mb[g]=f),e}});var nb=/^(?:input|select|textarea|button)$/i,ob=/^(?:a|area)$/i;r.fn.extend({prop:function(a,b){return T(this,r.prop,a,b,arguments.length>1)},removeProp:function(a){return this.each(function(){delete this[r.propFix[a]||a]})}}),r.extend({prop:function(a,b,c){var d,e,f=a.nodeType;if(3!==f&&8!==f&&2!==f)return 1===f&&r.isXMLDoc(a)||(b=r.propFix[b]||b,e=r.propHooks[b]),void 0!==c?e&&"set"in e&&void 0!==(d=e.set(a,c,b))?d:a[b]=c:e&&"get"in e&
&null!==(d=e.get(a,b))?d:a[b]},propHooks:{tabIndex:{get:function(a){var b=r.find.attr(a,"tabindex");return b?parseInt(b,10):nb.test(a.nodeName)||ob.test(a.nodeName)&&a.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),o.optSelected||(r.propHooks.selected={get:function(a){var b=a.parentNode;return b&&b.parentNode&&b.parentNode.selectedIndex,null},set:function(a){var b=a.parentNode;b&&(b.selectedIndex,b.parentNode&&b.parentNode.selectedIndex)}}),r.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){r.propFix[this.toLowerCase()]=this});function pb(a){var b=a.match(L)||[];return b.join(" ")}function qb(a){return a.getAttribute&&a.getAttribute("class")||""}r.fn.extend({addClass:function(a){var b,c,d,e,f,g,h,i=0;if(r.isFunction(a))return this.each(function(b){r(this).addClass(a.call(this,b,qb(this)))});if("string"==typeof a&&a){b=a.match(L)||[];while(c=this[i++])if(e=qb(c),d=1===c.nodeType&
&" "+pb(e)+" "){g=0;while(f=b[g++])d.indexOf(" "+f+" ")<0&&(d+=f+" ");h=pb(d),e!==h&&c.setAttribute("class",h)}}return this},removeClass:function(a){var b,c,d,e,f,g,h,i=0;if(r.isFunction(a))return this.each(function(b){r(this).removeClass(a.call(this,b,qb(this)))});if(!arguments.length)return this.attr("class","");if("string"==typeof a&&a){b=a.match(L)||[];while(c=this[i++])if(e=qb(c),d=1===c.nodeType&&" "+pb(e)+" "){g=0;while(f=b[g++])while(d.indexOf(" "+f+" ")>-1)d=d.replace(" "+f+" "," ");h=pb(d),e!==h&&c.setAttribute("class",h)}}return this},toggleClass:function(a,b){var c=typeof a;return"boolean"==typeof b&&"string"===c?b?this.addClass(a):this.removeClass(a):r.isFunction(a)?this.each(function(c){r(this).toggleClass(a.call(this,c,qb(this),b),b)}):this.each(function(){var b,d,e,f;if("string"===c){d=0,e=r(this),f=a.match(L)||[];while(b=f[d++])e.hasClass(b)?e.removeClass(b):e.addClass(b)}else void 0!==a&&"boolean"!==c||(b=qb(this),b&&W.set(this,"__className__",b),this.setAttribute&
&this.setAttribute("class",b||a===!1?"":W.get(this,"__className__")||""))})},hasClass:function(a){var b,c,d=0;b=" "+a+" ";while(c=this[d++])if(1===c.nodeType&&(" "+pb(qb(c))+" ").indexOf(b)>-1)return!0;return!1}});var rb=/\r/g;r.fn.extend({val:function(a){var b,c,d,e=this[0];{if(arguments.length)return d=r.isFunction(a),this.each(function(c){var e;1===this.nodeType&&(e=d?a.call(this,c,r(this).val()):a,null==e?e="":"number"==typeof e?e+="":Array.isArray(e)&&(e=r.map(e,function(a){return null==a?"":a+""})),b=r.valHooks[this.type]||r.valHooks[this.nodeName.toLowerCase()],b&&"set"in b&&void 0!==b.set(this,e,"value")||(this.value=e))});if(e)return b=r.valHooks[e.type]||r.valHooks[e.nodeName.toLowerCase()],b&&"get"in b&&void 0!==(c=b.get(e,"value"))?c:(c=e.value,"string"==typeof c?c.replace(rb,""):null==c?"":c)}}}),r.extend({valHooks:{option:{get:function(a){var b=r.find.attr(a,"value");return null!=b?b:pb(r.text(a))}},select:{get:function(a){var b,c,d,e=a.options,f=a.selectedIndex,g="sel
ect-one"===a.type,h=g?null:[],i=g?f+1:e.length;for(d=f<0?i:g?f:0;d<i;d++)if(c=e[d],(c.selected||d===f)&&!c.disabled&&(!c.parentNode.disabled||!B(c.parentNode,"optgroup"))){if(b=r(c).val(),g)return b;h.push(b)}return h},set:function(a,b){var c,d,e=a.options,f=r.makeArray(b),g=e.length;while(g--)d=e[g],(d.selected=r.inArray(r.valHooks.option.get(d),f)>-1)&&(c=!0);return c||(a.selectedIndex=-1),f}}}}),r.each(["radio","checkbox"],function(){r.valHooks[this]={set:function(a,b){if(Array.isArray(b))return a.checked=r.inArray(r(a).val(),b)>-1}},o.checkOn||(r.valHooks[this].get=function(a){return null===a.getAttribute("value")?"on":a.value})});var sb=/^(?:focusinfocus|focusoutblur)$/;r.extend(r.event,{trigger:function(b,c,e,f){var g,h,i,j,k,m,n,o=[e||d],p=l.call(b,"type")?b.type:b,q=l.call(b,"namespace")?b.namespace.split("."):[];if(h=i=e=e||d,3!==e.nodeType&&8!==e.nodeType&&!sb.test(p+r.event.triggered)&&(p.indexOf(".")>-1&&(q=p.split("."),p=q.shift(),q.sort()),k=p.indexOf(":")<0&&"on"+p,b=
b[r.expando]?b:new r.Event(p,"object"==typeof b&&b),b.isTrigger=f?2:3,b.namespace=q.join("."),b.rnamespace=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=e),c=null==c?[b]:r.makeArray(c,[b]),n=r.event.special[p]||{},f||!n.trigger||n.trigger.apply(e,c)!==!1)){if(!f&&!n.noBubble&&!r.isWindow(e)){for(j=n.delegateType||p,sb.test(j+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),i=h;i===(e.ownerDocument||d)&&o.push(i.defaultView||i.parentWindow||a)}g=0;while((h=o[g++])&&!b.isPropagationStopped())b.type=g>1?j:n.bindType||p,m=(W.get(h,"events")||{})[b.type]&&W.get(h,"handle"),m&&m.apply(h,c),m=k&&h[k],m&&m.apply&&U(h)&&(b.result=m.apply(h,c),b.result===!1&&b.preventDefault());return b.type=p,f||b.isDefaultPrevented()||n._default&&n._default.apply(o.pop(),c)!==!1||!U(e)||k&&r.isFunction(e[p])&&!r.isWindow(e)&&(i=e[k],i&&(e[k]=null),r.event.triggered=p,e[p](),r.event.triggered=void 0,i&&(e[k]=i)),b.result}},simulate:function(a,b,c)
{var d=r.extend(new r.Event,c,{type:a,isSimulated:!0});r.event.trigger(d,null,b)}}),r.fn.extend({trigger:function(a,b){return this.each(function(){r.event.trigger(a,b,this)})},triggerHandler:function(a,b){var c=this[0];if(c)return r.event.trigger(a,b,c,!0)}}),r.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(a,b){r.fn[b]=function(a,c){return arguments.length>0?this.on(b,null,a,c):this.trigger(b)}}),r.fn.extend({hover:function(a,b){return this.mouseenter(a).mouseleave(b||a)}}),o.focusin="onfocusin"in a,o.focusin||r.each({focus:"focusin",blur:"focusout"},function(a,b){var c=function(a){r.event.simulate(b,a.target,r.event.fix(a))};r.event.special[b]={setup:function(){var d=this.ownerDocument||this,e=W.access(d,b);e||d.addEventListener(a,c,!0),W.access(d,b,(e||0)+1)},teardown:function(){var d=this.ownerDocument||this,e=W.access(
d,b)-1;e?W.access(d,b,e):(d.removeEventListener(a,c,!0),W.remove(d,b))}}});var tb=a.location,ub=r.now(),vb=/\?/;r.parseXML=function(b){var c;if(!b||"string"!=typeof b)return null;try{c=(new a.DOMParser).parseFromString(b,"text/xml")}catch(d){c=void 0}return c&&!c.getElementsByTagName("parsererror").length||r.error("Invalid XML: "+b),c};var wb=/\[\]$/,xb=/\r?\n/g,yb=/^(?:submit|button|image|reset|file)$/i,zb=/^(?:input|select|textarea|keygen)/i;function Ab(a,b,c,d){var e;if(Array.isArray(b))r.each(b,function(b,e){c||wb.test(a)?d(a,e):Ab(a+"["+("object"==typeof e&&null!=e?b:"")+"]",e,c,d)});else if(c||"object"!==r.type(b))d(a,b);else for(e in b)Ab(a+"["+e+"]",b[e],c,d)}r.param=function(a,b){var c,d=[],e=function(a,b){var c=r.isFunction(b)?b():b;d[d.length]=encodeURIComponent(a)+"="+encodeURIComponent(null==c?"":c)};if(Array.isArray(a)||a.jquery&&!r.isPlainObject(a))r.each(a,function(){e(this.name,this.value)});else for(c in a)Ab(c,a[c],b,e);return d.join("&")},r.fn.extend({serialize:f
unction(){return r.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var a=r.prop(this,"elements");return a?r.makeArray(a):this}).filter(function(){var a=this.type;return this.name&&!r(this).is(":disabled")&&zb.test(this.nodeName)&&!yb.test(a)&&(this.checked||!ja.test(a))}).map(function(a,b){var c=r(this).val();return null==c?null:Array.isArray(c)?r.map(c,function(a){return{name:b.name,value:a.replace(xb,"\r\n")}}):{name:b.name,value:c.replace(xb,"\r\n")}}).get()}});var Bb=/%20/g,Cb=/#.*$/,Db=/([?&])_=[^&]*/,Eb=/^(.*?):[ \t]*([^\r\n]*)$/gm,Fb=/^(?:about|app|app-storage|.+-extension|file|res|widget):$/,Gb=/^(?:GET|HEAD)$/,Hb=/^\/\//,Ib={},Jb={},Kb="*/".concat("*"),Lb=d.createElement("a");Lb.href=tb.href;function Mb(a){return function(b,c){"string"!=typeof b&&(c=b,b="*");var d,e=0,f=b.toLowerCase().match(L)||[];if(r.isFunction(c))while(d=f[e++])"+"===d[0]?(d=d.slice(1)||"*",(a[d]=a[d]||[]).unshift(c)):(a[d]=a[d]||[]).push(c)}}function Nb(a,b,c,d){var e
={},f=a===Jb;function g(h){var i;return e[h]=!0,r.each(a[h]||[],function(a,h){var j=h(b,c,d);return"string"!=typeof j||f||e[j]?f?!(i=j):void 0:(b.dataTypes.unshift(j),g(j),!1)}),i}return g(b.dataTypes[0])||!e["*"]&&g("*")}function Ob(a,b){var c,d,e=r.ajaxSettings.flatOptions||{};for(c in b)void 0!==b[c]&&((e[c]?a:d||(d={}))[c]=b[c]);return d&&r.extend(!0,a,d),a}function Pb(a,b,c){var d,e,f,g,h=a.contents,i=a.dataTypes;while("*"===i[0])i.shift(),void 0===d&&(d=a.mimeType||b.getResponseHeader("Content-Type"));if(d)for(e in h)if(h[e]&&h[e].test(d)){i.unshift(e);break}if(i[0]in c)f=i[0];else{for(e in c){if(!i[0]||a.converters[e+" "+i[0]]){f=e;break}g||(g=e)}f=f||g}if(f)return f!==i[0]&&i.unshift(f),c[f]}function Qb(a,b,c,d){var e,f,g,h,i,j={},k=a.dataTypes.slice();if(k[1])for(g in a.converters)j[g.toLowerCase()]=a.converters[g];f=k.shift();while(f)if(a.responseFields[f]&&(c[a.responseFields[f]]=b),!i&&d&&a.dataFilter&&(b=a.dataFilter(b,a.dataType)),i=f,f=k.shift())if("*"===f)f=i;else if
("*"!==i&&i!==f){if(g=j[i+" "+f]||j["* "+f],!g)for(e in j)if(h=e.split(" "),h[1]===f&&(g=j[i+" "+h[0]]||j["* "+h[0]])){g===!0?g=j[e]:j[e]!==!0&&(f=h[0],k.unshift(h[1]));break}if(g!==!0)if(g&&a["throws"])b=g(b);else try{b=g(b)}catch(l){return{state:"parsererror",error:g?l:"No conversion from "+i+" to "+f}}}return{state:"success",data:b}}r.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:tb.href,type:"GET",isLocal:Fb.test(tb.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":Kb,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":r.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(a,b){return b?Ob(Ob(a,r.ajaxSettings),b):Ob(r.ajaxSetting
s,a)},ajaxPrefilter:Mb(Ib),ajaxTransport:Mb(Jb),ajax:function(b,c){"object"==typeof b&&(c=b,b=void 0),c=c||{};var e,f,g,h,i,j,k,l,m,n,o=r.ajaxSetup({},c),p=o.context||o,q=o.context&&(p.nodeType||p.jquery)?r(p):r.event,s=r.Deferred(),t=r.Callbacks("once memory"),u=o.statusCode||{},v={},w={},x="canceled",y={readyState:0,getResponseHeader:function(a){var b;if(k){if(!h){h={};while(b=Eb.exec(g))h[b[1].toLowerCase()]=b[2]}b=h[a.toLowerCase()]}return null==b?null:b},getAllResponseHeaders:function(){return k?g:null},setRequestHeader:function(a,b){return null==k&&(a=w[a.toLowerCase()]=w[a.toLowerCase()]||a,v[a]=b),this},overrideMimeType:function(a){return null==k&&(o.mimeType=a),this},statusCode:function(a){var b;if(a)if(k)y.always(a[y.status]);else for(b in a)u[b]=[u[b],a[b]];return this},abort:function(a){var b=a||x;return e&&e.abort(b),A(0,b),this}};if(s.promise(y),o.url=((b||o.url||tb.href)+"").replace(Hb,tb.protocol+"//"),o.type=c.method||c.type||o.method||o.type,o.dataTypes=(o.dataType
||"*").toLowerCase().match(L)||[""],null==o.crossDomain){j=d.createElement("a");try{j.href=o.url,j.href=j.href,o.crossDomain=Lb.protocol+"//"+Lb.host!=j.protocol+"//"+j.host}catch(z){o.crossDomain=!0}}if(o.data&&o.processData&&"string"!=typeof o.data&&(o.data=r.param(o.data,o.traditional)),Nb(Ib,o,c,y),k)return y;l=r.event&&o.global,l&&0===r.active++&&r.event.trigger("ajaxStart"),o.type=o.type.toUpperCase(),o.hasContent=!Gb.test(o.type),f=o.url.replace(Cb,""),o.hasContent?o.data&&o.processData&&0===(o.contentType||"").indexOf("application/x-www-form-urlencoded")&&(o.data=o.data.replace(Bb,"+")):(n=o.url.slice(f.length),o.data&&(f+=(vb.test(f)?"&":"?")+o.data,delete o.data),o.cache===!1&&(f=f.replace(Db,"$1"),n=(vb.test(f)?"&":"?")+"_="+ub++ +n),o.url=f+n),o.ifModified&&(r.lastModified[f]&&y.setRequestHeader("If-Modified-Since",r.lastModified[f]),r.etag[f]&&y.setRequestHeader("If-None-Match",r.etag[f])),(o.data&&o.hasContent&&o.contentType!==!1||c.contentType)&&y.setRequestHeader("Co
ntent-Type",o.contentType),y.setRequestHeader("Accept",o.dataTypes[0]&&o.accepts[o.dataTypes[0]]?o.accepts[o.dataTypes[0]]+("*"!==o.dataTypes[0]?", "+Kb+"; q=0.01":""):o.accepts["*"]);for(m in o.headers)y.setRequestHeader(m,o.headers[m]);if(o.beforeSend&&(o.beforeSend.call(p,y,o)===!1||k))return y.abort();if(x="abort",t.add(o.complete),y.done(o.success),y.fail(o.error),e=Nb(Jb,o,c,y)){if(y.readyState=1,l&&q.trigger("ajaxSend",[y,o]),k)return y;o.async&&o.timeout>0&&(i=a.setTimeout(function(){y.abort("timeout")},o.timeout));try{k=!1,e.send(v,A)}catch(z){if(k)throw z;A(-1,z)}}else A(-1,"No Transport");function A(b,c,d,h){var j,m,n,v,w,x=c;k||(k=!0,i&&a.clearTimeout(i),e=void 0,g=h||"",y.readyState=b>0?4:0,j=b>=200&&b<300||304===b,d&&(v=Pb(o,y,d)),v=Qb(o,v,y,j),j?(o.ifModified&&(w=y.getResponseHeader("Last-Modified"),w&&(r.lastModified[f]=w),w=y.getResponseHeader("etag"),w&&(r.etag[f]=w)),204===b||"HEAD"===o.type?x="nocontent":304===b?x="notmodified":(x=v.state,m=v.data,n=v.error,j=!n)
):(n=x,!b&&x||(x="error",b<0&&(b=0))),y.status=b,y.statusText=(c||x)+"",j?s.resolveWith(p,[m,x,y]):s.rejectWith(p,[y,x,n]),y.statusCode(u),u=void 0,l&&q.trigger(j?"ajaxSuccess":"ajaxError",[y,o,j?m:n]),t.fireWith(p,[y,x]),l&&(q.trigger("ajaxComplete",[y,o]),--r.active||r.event.trigger("ajaxStop")))}return y},getJSON:function(a,b,c){return r.get(a,b,c,"json")},getScript:function(a,b){return r.get(a,void 0,b,"script")}}),r.each(["get","post"],function(a,b){r[b]=function(a,c,d,e){return r.isFunction(c)&&(e=e||d,d=c,c=void 0),r.ajax(r.extend({url:a,type:b,dataType:e,data:c,success:d},r.isPlainObject(a)&&a))}}),r._evalUrl=function(a){return r.ajax({url:a,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,"throws":!0})},r.fn.extend({wrapAll:function(a){var b;return this[0]&&(r.isFunction(a)&&(a=a.call(this[0])),b=r(a,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&b.insertBefore(this[0]),b.map(function(){var a=this;while(a.firstElementChild)a=a.firstElementChild;return a}
).append(this)),this},wrapInner:function(a){return r.isFunction(a)?this.each(function(b){r(this).wrapInner(a.call(this,b))}):this.each(function(){var b=r(this),c=b.contents();c.length?c.wrapAll(a):b.append(a)})},wrap:function(a){var b=r.isFunction(a);return this.each(function(c){r(this).wrapAll(b?a.call(this,c):a)})},unwrap:function(a){return this.parent(a).not("body").each(function(){r(this).replaceWith(this.childNodes)}),this}}),r.expr.pseudos.hidden=function(a){return!r.expr.pseudos.visible(a)},r.expr.pseudos.visible=function(a){return!!(a.offsetWidth||a.offsetHeight||a.getClientRects().length)},r.ajaxSettings.xhr=function(){try{return new a.XMLHttpRequest}catch(b){}};var Rb={0:200,1223:204},Sb=r.ajaxSettings.xhr();o.cors=!!Sb&&"withCredentials"in Sb,o.ajax=Sb=!!Sb,r.ajaxTransport(function(b){var c,d;if(o.cors||Sb&&!b.crossDomain)return{send:function(e,f){var g,h=b.xhr();if(h.open(b.type,b.url,b.async,b.username,b.password),b.xhrFields)for(g in b.xhrFields)h[g]=b.xhrFields[g];b.m
imeType&&h.overrideMimeType&&h.overrideMimeType(b.mimeType),b.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest");for(g in e)h.setRequestHeader(g,e[g]);c=function(a){return function(){c&&(c=d=h.onload=h.onerror=h.onabort=h.onreadystatechange=null,"abort"===a?h.abort():"error"===a?"number"!=typeof h.status?f(0,"error"):f(h.status,h.statusText):f(Rb[h.status]||h.status,h.statusText,"text"!==(h.responseType||"text")||"string"!=typeof h.responseText?{binary:h.response}:{text:h.responseText},h.getAllResponseHeaders()))}},h.onload=c(),d=h.onerror=c("error"),void 0!==h.onabort?h.onabort=d:h.onreadystatechange=function(){4===h.readyState&&a.setTimeout(function(){c&&d()})},c=c("abort");try{h.send(b.hasContent&&b.data||null)}catch(i){if(c)throw i}},abort:function(){c&&c()}}}),r.ajaxPrefilter(function(a){a.crossDomain&&(a.contents.script=!1)}),r.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},conte
nts:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(a){return r.globalEval(a),a}}}),r.ajaxPrefilter("script",function(a){void 0===a.cache&&(a.cache=!1),a.crossDomain&&(a.type="GET")}),r.ajaxTransport("script",function(a){if(a.crossDomain){var b,c;return{send:function(e,f){b=r("<script>").prop({charset:a.scriptCharset,src:a.url}).on("load error",c=function(a){b.remove(),c=null,a&&f("error"===a.type?404:200,a.type)}),d.head.appendChild(b[0])},abort:function(){c&&c()}}}});var Tb=[],Ub=/(=)\?(?=&|$)|\?\?/;r.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var a=Tb.pop()||r.expando+"_"+ub++;return this[a]=!0,a}}),r.ajaxPrefilter("json jsonp",function(b,c,d){var e,f,g,h=b.jsonp!==!1&&(Ub.test(b.url)?"url":"string"==typeof b.data&&0===(b.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ub.test(b.data)&&"data");if(h||"jsonp"===b.dataTypes[0])return e=b.jsonpCallback=r.isFunction(b.jsonpCallback)?b.jsonpCallback():b.jsonpCallback,h?b[h]=b[h].replace(Ub
,"$1"+e):b.jsonp!==!1&&(b.url+=(vb.test(b.url)?"&":"?")+b.jsonp+"="+e),b.converters["script json"]=function(){return g||r.error(e+" was not called"),g[0]},b.dataTypes[0]="json",f=a[e],a[e]=function(){g=arguments},d.always(function(){void 0===f?r(a).removeProp(e):a[e]=f,b[e]&&(b.jsonpCallback=c.jsonpCallback,Tb.push(e)),g&&r.isFunction(f)&&f(g[0]),g=f=void 0}),"script"}),o.createHTMLDocument=function(){var a=d.implementation.createHTMLDocument("").body;return a.innerHTML="<form></form><form></form>",2===a.childNodes.length}(),r.parseHTML=function(a,b,c){if("string"!=typeof a)return[];"boolean"==typeof b&&(c=b,b=!1);var e,f,g;return b||(o.createHTMLDocument?(b=d.implementation.createHTMLDocument(""),e=b.createElement("base"),e.href=d.location.href,b.head.appendChild(e)):b=d),f=C.exec(a),g=!c&&[],f?[b.createElement(f[1])]:(f=qa([a],b,g),g&&g.length&&r(g).remove(),r.merge([],f.childNodes))},r.fn.load=function(a,b,c){var d,e,f,g=this,h=a.indexOf(" ");return h>-1&&(d=pb(a.slice(h)),a=a.sl
ice(0,h)),r.isFunction(b)?(c=b,b=void 0):b&&"object"==typeof b&&(e="POST"),g.length>0&&r.ajax({url:a,type:e||"GET",dataType:"html",data:b}).done(function(a){f=arguments,g.html(d?r("<div>").append(r.parseHTML(a)).find(d):a)}).always(c&&function(a,b){g.each(function(){c.apply(this,f||[a.responseText,b,a])})}),this},r.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(a,b){r.fn[b]=function(a){return this.on(b,a)}}),r.expr.pseudos.animated=function(a){return r.grep(r.timers,function(b){return a===b.elem}).length},r.offset={setOffset:function(a,b,c){var d,e,f,g,h,i,j,k=r.css(a,"position"),l=r(a),m={};"static"===k&&(a.style.position="relative"),h=l.offset(),f=r.css(a,"top"),i=r.css(a,"left"),j=("absolute"===k||"fixed"===k)&&(f+i).indexOf("auto")>-1,j?(d=l.position(),g=d.top,e=d.left):(g=parseFloat(f)||0,e=parseFloat(i)||0),r.isFunction(b)&&(b=b.call(a,c,r.extend({},h))),null!=b.top&&(m.top=b.top-h.top+g),null!=b.left&&(m.left=b.left-h.left+e),"using
"in b?b.using.call(a,m):l.css(m)}},r.fn.extend({offset:function(a){if(arguments.length)return void 0===a?this:this.each(function(b){r.offset.setOffset(this,a,b)});var b,c,d,e,f=this[0];if(f)return f.getClientRects().length?(d=f.getBoundingClientRect(),b=f.ownerDocument,c=b.documentElement,e=b.defaultView,{top:d.top+e.pageYOffset-c.clientTop,left:d.left+e.pageXOffset-c.clientLeft}):{top:0,left:0}},position:function(){if(this[0]){var a,b,c=this[0],d={top:0,left:0};return"fixed"===r.css(c,"position")?b=c.getBoundingClientRect():(a=this.offsetParent(),b=this.offset(),B(a[0],"html")||(d=a.offset()),d={top:d.top+r.css(a[0],"borderTopWidth",!0),left:d.left+r.css(a[0],"borderLeftWidth",!0)}),{top:b.top-d.top-r.css(c,"marginTop",!0),left:b.left-d.left-r.css(c,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var a=this.offsetParent;while(a&&"static"===r.css(a,"position"))a=a.offsetParent;return a||ra})}}),r.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function
(a,b){var c="pageYOffset"===b;r.fn[a]=function(d){return T(this,function(a,d,e){var f;return r.isWindow(a)?f=a:9===a.nodeType&&(f=a.defaultView),void 0===e?f?f[b]:a[d]:void(f?f.scrollTo(c?f.pageXOffset:e,c?e:f.pageYOffset):a[d]=e)},a,d,arguments.length)}}),r.each(["top","left"],function(a,b){r.cssHooks[b]=Pa(o.pixelPosition,function(a,c){if(c)return c=Oa(a,b),Ma.test(c)?r(a).position()[b]+"px":c})}),r.each({Height:"height",Width:"width"},function(a,b){r.each({padding:"inner"+a,content:b,"":"outer"+a},function(c,d){r.fn[d]=function(e,f){var g=arguments.length&&(c||"boolean"!=typeof e),h=c||(e===!0||f===!0?"margin":"border");return T(this,function(b,c,e){var f;return r.isWindow(b)?0===d.indexOf("outer")?b["inner"+a]:b.document.documentElement["client"+a]:9===b.nodeType?(f=b.documentElement,Math.max(b.body["scroll"+a],f["scroll"+a],b.body["offset"+a],f["offset"+a],f["client"+a])):void 0===e?r.css(b,c,h):r.style(b,c,e,h)},b,g?e:void 0,g)}})}),r.fn.extend({bind:function(a,b,c){return thi
s.on(a,null,b,c)},unbind:function(a,b){return this.off(a,null,b)},delegate:function(a,b,c,d){return this.on(b,a,c,d)},undelegate:function(a,b,c){return 1===arguments.length?this.off(a,"**"):this.off(b,a||"**",c)}}),r.holdReady=function(a){a?r.readyWait++:r.ready(!0)},r.isArray=Array.isArray,r.parseJSON=JSON.parse,r.nodeName=B,"function"==typeof define&&define.amd&&define("jquery",[],function(){return r});var Vb=a.jQuery,Wb=a.$;return r.noConflict=function(b){return a.$===r&&(a.$=Wb),b&&a.jQuery===r&&(a.jQuery=Vb),r},b||(a.jQuery=a.$=r),r});
[02/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/recovery/BlackholeStateStoreSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/recovery/BlackholeStateStoreSpec.scala b/server/src/test/scala/org/apache/livy/server/recovery/BlackholeStateStoreSpec.scala
new file mode 100644
index 0000000..e40bb1c
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/recovery/BlackholeStateStoreSpec.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.recovery
+
+import org.scalatest.FunSpec
+import org.scalatest.Matchers._
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+
+class BlackholeStateStoreSpec extends FunSpec with LivyBaseUnitTestSuite {
+ describe("BlackholeStateStore") {
+ val stateStore = new BlackholeStateStore(new LivyConf())
+
+ it("set should not throw") {
+ stateStore.set("", 1.asInstanceOf[Object])
+ }
+
+ it("get should return None") {
+ val v = stateStore.get[Object]("")
+ v shouldBe None
+ }
+
+ it("getChildren should return empty list") {
+ val c = stateStore.getChildren("")
+ c shouldBe empty
+ }
+
+ it("remove should not throw") {
+ stateStore.remove("")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/recovery/FileSystemStateStoreSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/recovery/FileSystemStateStoreSpec.scala b/server/src/test/scala/org/apache/livy/server/recovery/FileSystemStateStoreSpec.scala
new file mode 100644
index 0000000..4758c85
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/recovery/FileSystemStateStoreSpec.scala
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.recovery
+
+import java.io.{FileNotFoundException, InputStream, IOException}
+import java.util
+
+import org.apache.hadoop.fs._
+import org.apache.hadoop.fs.Options.{CreateOpts, Rename}
+import org.apache.hadoop.fs.permission.FsPermission
+import org.hamcrest.Description
+import org.mockito.ArgumentMatcher
+import org.mockito.Matchers.{any, anyInt, argThat, eq => equal}
+import org.mockito.Mockito.{atLeastOnce, verify, when}
+import org.mockito.internal.matchers.Equals
+import org.mockito.invocation.InvocationOnMock
+import org.mockito.stubbing.Answer
+import org.scalatest.FunSpec
+import org.scalatest.Matchers._
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+
+class FileSystemStateStoreSpec extends FunSpec with LivyBaseUnitTestSuite {
+ describe("FileSystemStateStore") {
+ def pathEq(wantedPath: String): Path = argThat(new ArgumentMatcher[Path] {
+ private val matcher = new Equals(wantedPath)
+
+ override def matches(path: Any): Boolean = matcher.matches(path.toString)
+
+ override def describeTo(d: Description): Unit = { matcher.describeTo(d) }
+ })
+
+ def makeConf(): LivyConf = {
+ val conf = new LivyConf()
+ conf.set(LivyConf.RECOVERY_STATE_STORE_URL, "file://tmp/")
+
+ conf
+ }
+
+ def mockFileContext(rootDirPermission: String): FileContext = {
+ val fileContext = mock[FileContext]
+ val rootDirStatus = mock[FileStatus]
+ when(fileContext.getFileStatus(any())).thenReturn(rootDirStatus)
+ when(rootDirStatus.getPermission).thenReturn(new FsPermission(rootDirPermission))
+
+ fileContext
+ }
+
+ it("should throw if url is not configured") {
+ intercept[IllegalArgumentException](new FileSystemStateStore(new LivyConf()))
+ }
+
+ it("should set and verify file permission") {
+ val fileContext = mockFileContext("700")
+ new FileSystemStateStore(makeConf(), Some(fileContext))
+
+ verify(fileContext).setUMask(new FsPermission("077"))
+ }
+
+ it("should reject insecure permission") {
+ def test(permission: String): Unit = {
+ val fileContext = mockFileContext(permission)
+
+ intercept[IllegalArgumentException](new FileSystemStateStore(makeConf(), Some(fileContext)))
+ }
+ test("600")
+ test("400")
+ test("677")
+ test("670")
+ test("607")
+ }
+
+ it("set should write with an intermediate file") {
+ val fileContext = mockFileContext("700")
+ val outputStream = mock[FSDataOutputStream]
+ when(fileContext.create(pathEq("/key.tmp"), any[util.EnumSet[CreateFlag]], any[CreateOpts]))
+ .thenReturn(outputStream)
+
+ val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
+
+ stateStore.set("key", "value")
+
+ verify(outputStream).write(""""value"""".getBytes)
+ verify(outputStream, atLeastOnce).close()
+
+
+ verify(fileContext).rename(pathEq("/key.tmp"), pathEq("/key"), equal(Rename.OVERWRITE))
+ verify(fileContext).delete(pathEq("/.key.tmp.crc"), equal(false))
+ }
+
+ it("get should read file") {
+ val fileContext = mockFileContext("700")
+ abstract class MockInputStream extends InputStream with Seekable with PositionedReadable {}
+ val inputStream: InputStream = mock[MockInputStream]
+ when(inputStream.read(any[Array[Byte]](), anyInt(), anyInt())).thenAnswer(new Answer[Int] {
+ private var firstCall = true
+ override def answer(invocation: InvocationOnMock): Int = {
+ if (firstCall) {
+ firstCall = false
+ val buf = invocation.getArguments()(0).asInstanceOf[Array[Byte]]
+ val b = """"value"""".getBytes()
+ b.copyToArray(buf)
+ b.length
+ } else {
+ -1
+ }
+ }
+ })
+
+ when(fileContext.open(pathEq("/key"))).thenReturn(new FSDataInputStream(inputStream))
+
+ val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
+
+ stateStore.get[String]("key") shouldBe Some("value")
+
+ verify(inputStream, atLeastOnce).close()
+ }
+
+ it("get non-existent key should return None") {
+ val fileContext = mockFileContext("700")
+ when(fileContext.open(any())).thenThrow(new FileNotFoundException("Unit test"))
+
+ val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
+
+ stateStore.get[String]("key") shouldBe None
+ }
+
+ it("getChildren should list file") {
+ val parentPath = "path"
+ def makeFileStatus(name: String): FileStatus = {
+ val fs = new FileStatus()
+ fs.setPath(new Path(parentPath, name))
+ fs
+ }
+ val children = Seq("c1", "c2")
+
+ val fileContext = mockFileContext("700")
+ val util = mock[FileContext#Util]
+ when(util.listStatus(pathEq(s"/$parentPath")))
+ .thenReturn(children.map(makeFileStatus).toArray)
+ when(fileContext.util()).thenReturn(util)
+
+ val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
+ stateStore.getChildren(parentPath) should contain theSameElementsAs children
+ }
+
+ def getChildrenErrorTest(error: Exception): Unit = {
+ val parentPath = "path"
+
+ val fileContext = mockFileContext("700")
+ val util = mock[FileContext#Util]
+ when(util.listStatus(pathEq(s"/$parentPath"))).thenThrow(error)
+ when(fileContext.util()).thenReturn(util)
+
+ val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
+ stateStore.getChildren(parentPath) shouldBe empty
+ }
+
+ it("getChildren should return empty list if the key doesn't exist") {
+ getChildrenErrorTest(new IOException("Unit test"))
+ }
+
+ it("getChildren should return empty list if key doesn't exist") {
+ getChildrenErrorTest(new FileNotFoundException("Unit test"))
+ }
+
+ it("remove should delete file") {
+ val fileContext = mockFileContext("700")
+
+ val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
+ stateStore.remove("key")
+
+ verify(fileContext).delete(pathEq("/key"), equal(false))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/recovery/SessionStoreSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/recovery/SessionStoreSpec.scala b/server/src/test/scala/org/apache/livy/server/recovery/SessionStoreSpec.scala
new file mode 100644
index 0000000..5eeb2cf
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/recovery/SessionStoreSpec.scala
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.recovery
+
+import scala.util.Success
+
+import org.mockito.Mockito._
+import org.scalatest.FunSpec
+import org.scalatest.Matchers._
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+import org.apache.livy.sessions.Session.RecoveryMetadata
+
+class SessionStoreSpec extends FunSpec with LivyBaseUnitTestSuite {
+ describe("SessionStore") {
+ case class TestRecoveryMetadata(id: Int) extends RecoveryMetadata
+
+ val sessionType = "test"
+ val sessionPath = s"v1/$sessionType"
+ val sessionManagerPath = s"v1/$sessionType/state"
+
+ val conf = new LivyConf()
+ it("should set session state and session counter when saving a session.") {
+ val stateStore = mock[StateStore]
+ val sessionStore = new SessionStore(conf, stateStore)
+
+ val m = TestRecoveryMetadata(99)
+ sessionStore.save(sessionType, m)
+ verify(stateStore).set(s"$sessionPath/99", m)
+ }
+
+ it("should return existing sessions") {
+ val validMetadata = Map(
+ "0" -> Some(TestRecoveryMetadata(0)),
+ "5" -> None,
+ "77" -> Some(TestRecoveryMetadata(77)))
+ val corruptedMetadata = Map(
+ "7" -> new RuntimeException("Test"),
+ "11212" -> new RuntimeException("Test")
+ )
+ val stateStore = mock[StateStore]
+ val sessionStore = new SessionStore(conf, stateStore)
+ when(stateStore.getChildren(sessionPath))
+ .thenReturn((validMetadata ++ corruptedMetadata).keys.toList)
+
+ validMetadata.foreach { case (id, m) =>
+ when(stateStore.get[TestRecoveryMetadata](s"$sessionPath/$id")).thenReturn(m)
+ }
+
+ corruptedMetadata.foreach { case (id, ex) =>
+ when(stateStore.get[TestRecoveryMetadata](s"$sessionPath/$id")).thenThrow(ex)
+ }
+
+ val s = sessionStore.getAllSessions[TestRecoveryMetadata](sessionType)
+ // Verify normal metadata are retrieved.
+ s.filter(_.isSuccess) should contain theSameElementsAs
+ validMetadata.values.filter(_.isDefined).map(m => Success(m.get))
+ // Verify exceptions are wrapped as in Try and are returned.
+ s.filter(_.isFailure) should have size corruptedMetadata.size
+ }
+
+ it("should not throw if the state store is empty") {
+ val stateStore = mock[StateStore]
+ val sessionStore = new SessionStore(conf, stateStore)
+ when(stateStore.getChildren(sessionPath)).thenReturn(Seq.empty)
+
+ val s = sessionStore.getAllSessions[TestRecoveryMetadata](sessionType)
+ s.filter(_.isSuccess) shouldBe empty
+ }
+
+ it("should return correct next session id") {
+ val stateStore = mock[StateStore]
+ val sessionStore = new SessionStore(conf, stateStore)
+
+ when(stateStore.get[SessionManagerState](sessionManagerPath)).thenReturn(None)
+ sessionStore.getNextSessionId(sessionType) shouldBe 0
+
+ val sms = SessionManagerState(100)
+ when(stateStore.get[SessionManagerState](sessionManagerPath)).thenReturn(Some(sms))
+ sessionStore.getNextSessionId(sessionType) shouldBe sms.nextSessionId
+ }
+
+ it("should remove session") {
+ val stateStore = mock[StateStore]
+ val sessionStore = new SessionStore(conf, stateStore)
+ val id = 1
+
+ sessionStore.remove(sessionType, 1)
+ verify(stateStore).remove(s"$sessionPath/$id")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/recovery/StateStoreSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/recovery/StateStoreSpec.scala b/server/src/test/scala/org/apache/livy/server/recovery/StateStoreSpec.scala
new file mode 100644
index 0000000..c7040a5
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/recovery/StateStoreSpec.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.recovery
+
+import scala.reflect.classTag
+
+import org.scalatest.{BeforeAndAfter, FunSpec}
+import org.scalatest.Matchers._
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+import org.apache.livy.sessions.SessionManager
+
+class StateStoreSpec extends FunSpec with BeforeAndAfter with LivyBaseUnitTestSuite {
+ describe("StateStore") {
+ after {
+ StateStore.cleanup()
+ }
+
+ def createConf(stateStore: String): LivyConf = {
+ val conf = new LivyConf()
+ conf.set(LivyConf.RECOVERY_MODE.key, SessionManager.SESSION_RECOVERY_MODE_RECOVERY)
+ conf.set(LivyConf.RECOVERY_STATE_STORE.key, stateStore)
+ conf
+ }
+
+ it("should throw an error on get if it's not initialized") {
+ intercept[AssertionError] { StateStore.get }
+ }
+
+ it("should initialize blackhole state store if recovery is disabled") {
+ StateStore.init(new LivyConf())
+ StateStore.get shouldBe a[BlackholeStateStore]
+ }
+
+ it("should pick the correct store according to state store config") {
+ StateStore.pickStateStore(createConf("filesystem")) shouldBe classOf[FileSystemStateStore]
+ StateStore.pickStateStore(createConf("zookeeper")) shouldBe classOf[ZooKeeperStateStore]
+ }
+
+ it("should return error if an unknown recovery mode is set") {
+ val conf = new LivyConf()
+ conf.set(LivyConf.RECOVERY_MODE.key, "unknown")
+ intercept[IllegalArgumentException] { StateStore.init(conf) }
+ }
+
+ it("should return error if an unknown state store is set") {
+ intercept[IllegalArgumentException] { StateStore.init(createConf("unknown")) }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/server/recovery/ZooKeeperStateStoreSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/server/recovery/ZooKeeperStateStoreSpec.scala b/server/src/test/scala/org/apache/livy/server/recovery/ZooKeeperStateStoreSpec.scala
new file mode 100644
index 0000000..88e530f
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/recovery/ZooKeeperStateStoreSpec.scala
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.recovery
+
+import scala.collection.JavaConverters._
+
+import org.apache.curator.framework.CuratorFramework
+import org.apache.curator.framework.api._
+import org.apache.curator.framework.listen.Listenable
+import org.apache.zookeeper.data.Stat
+import org.mockito.Mockito._
+import org.scalatest.FunSpec
+import org.scalatest.Matchers._
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+
+class ZooKeeperStateStoreSpec extends FunSpec with LivyBaseUnitTestSuite {
+ describe("ZooKeeperStateStore") {
+ case class TestFixture(stateStore: ZooKeeperStateStore, curatorClient: CuratorFramework)
+ val conf = new LivyConf()
+ conf.set(LivyConf.RECOVERY_STATE_STORE_URL, "host")
+ val key = "key"
+ val prefixedKey = s"/livy/$key"
+
+ def withMock[R](testBody: TestFixture => R): R = {
+ val curatorClient = mock[CuratorFramework]
+ when(curatorClient.getUnhandledErrorListenable())
+ .thenReturn(mock[Listenable[UnhandledErrorListener]])
+ val stateStore = new ZooKeeperStateStore(conf, Some(curatorClient))
+ testBody(TestFixture(stateStore, curatorClient))
+ }
+
+ def mockExistsBuilder(curatorClient: CuratorFramework, exists: Boolean): Unit = {
+ val existsBuilder = mock[ExistsBuilder]
+ when(curatorClient.checkExists()).thenReturn(existsBuilder)
+ if (exists) {
+ when(existsBuilder.forPath(prefixedKey)).thenReturn(mock[Stat])
+ }
+ }
+
+ it("should throw on bad config") {
+ withMock { f =>
+ val conf = new LivyConf()
+ intercept[IllegalArgumentException] { new ZooKeeperStateStore(conf) }
+
+ conf.set(LivyConf.RECOVERY_STATE_STORE_URL, "host")
+ conf.set(ZooKeeperStateStore.ZK_RETRY_CONF, "bad")
+ intercept[IllegalArgumentException] { new ZooKeeperStateStore(conf) }
+ }
+ }
+
+ it("set should use curatorClient") {
+ withMock { f =>
+ mockExistsBuilder(f.curatorClient, true)
+
+ val setDataBuilder = mock[SetDataBuilder]
+ when(f.curatorClient.setData()).thenReturn(setDataBuilder)
+
+ f.stateStore.set("key", 1.asInstanceOf[Object])
+
+ verify(f.curatorClient).start()
+ verify(setDataBuilder).forPath(prefixedKey, Array[Byte](49))
+ }
+ }
+
+ it("set should create parents if they don't exist") {
+ withMock { f =>
+ mockExistsBuilder(f.curatorClient, false)
+
+ val createBuilder = mock[CreateBuilder]
+ when(f.curatorClient.create()).thenReturn(createBuilder)
+ val p = mock[ProtectACLCreateModePathAndBytesable[String]]
+ when(createBuilder.creatingParentsIfNeeded()).thenReturn(p)
+
+ f.stateStore.set("key", 1.asInstanceOf[Object])
+
+ verify(f.curatorClient).start()
+ verify(p).forPath(prefixedKey, Array[Byte](49))
+ }
+ }
+
+ it("get should retrieve retry policy configs") {
+ conf.set(org.apache.livy.server.recovery.ZooKeeperStateStore.ZK_RETRY_CONF, "11,77")
+ withMock { f =>
+ mockExistsBuilder(f.curatorClient, true)
+
+ f.stateStore.retryPolicy should not be null
+ f.stateStore.retryPolicy.getN shouldBe 11
+ }
+ }
+
+ it("get should retrieve data from curatorClient") {
+ withMock { f =>
+ mockExistsBuilder(f.curatorClient, true)
+
+ val getDataBuilder = mock[GetDataBuilder]
+ when(f.curatorClient.getData()).thenReturn(getDataBuilder)
+ when(getDataBuilder.forPath(prefixedKey)).thenReturn(Array[Byte](50))
+
+ val v = f.stateStore.get[Int]("key")
+
+ verify(f.curatorClient).start()
+ v shouldBe Some(2)
+ }
+ }
+
+ it("get should return None if key doesn't exist") {
+ withMock { f =>
+ mockExistsBuilder(f.curatorClient, false)
+
+ val v = f.stateStore.get[Int]("key")
+
+ verify(f.curatorClient).start()
+ v shouldBe None
+ }
+ }
+
+ it("getChildren should use curatorClient") {
+ withMock { f =>
+ mockExistsBuilder(f.curatorClient, true)
+
+ val getChildrenBuilder = mock[GetChildrenBuilder]
+ when(f.curatorClient.getChildren()).thenReturn(getChildrenBuilder)
+ val children = List("abc", "def")
+ when(getChildrenBuilder.forPath(prefixedKey)).thenReturn(children.asJava)
+
+ val c = f.stateStore.getChildren("key")
+
+ verify(f.curatorClient).start()
+ c shouldBe children
+ }
+ }
+
+ it("getChildren should return empty list if key doesn't exist") {
+ withMock { f =>
+ mockExistsBuilder(f.curatorClient, false)
+
+ val c = f.stateStore.getChildren("key")
+
+ verify(f.curatorClient).start()
+ c shouldBe empty
+ }
+ }
+
+ it("remove should use curatorClient") {
+ withMock { f =>
+ val deleteBuilder = mock[DeleteBuilder]
+ when(f.curatorClient.delete()).thenReturn(deleteBuilder)
+ val g = mock[ChildrenDeletable]
+ when(deleteBuilder.guaranteed()).thenReturn(g)
+
+ f.stateStore.remove(key)
+
+ verify(g).forPath(prefixedKey)
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/sessions/MockSession.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/sessions/MockSession.scala b/server/src/test/scala/org/apache/livy/sessions/MockSession.scala
new file mode 100644
index 0000000..3cfbe46
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/sessions/MockSession.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.sessions
+
+import org.apache.livy.LivyConf
+
+class MockSession(id: Int, owner: String, conf: LivyConf) extends Session(id, owner, conf) {
+ case class RecoveryMetadata(id: Int) extends Session.RecoveryMetadata()
+
+ override val proxyUser = None
+
+ override protected def stopSession(): Unit = ()
+
+ override def logLines(): IndexedSeq[String] = IndexedSeq()
+
+ override def state: SessionState = SessionState.Idle()
+
+ override def recoveryMetadata: RecoveryMetadata = RecoveryMetadata(0)
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/sessions/SessionManagerSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/sessions/SessionManagerSpec.scala b/server/src/test/scala/org/apache/livy/sessions/SessionManagerSpec.scala
new file mode 100644
index 0000000..beffa71
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/sessions/SessionManagerSpec.scala
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.sessions
+
+import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.util.{Failure, Try}
+
+import org.mockito.Mockito.{doReturn, never, verify, when}
+import org.scalatest.{FunSpec, Matchers}
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+import org.apache.livy.server.batch.{BatchRecoveryMetadata, BatchSession}
+import org.apache.livy.server.interactive.InteractiveSession
+import org.apache.livy.server.recovery.SessionStore
+import org.apache.livy.sessions.Session.RecoveryMetadata
+
+class SessionManagerSpec extends FunSpec with Matchers with LivyBaseUnitTestSuite {
+ implicit def executor: ExecutionContext = ExecutionContext.global
+
+ describe("SessionManager") {
+ it("should garbage collect old sessions") {
+ val livyConf = new LivyConf()
+ livyConf.set(LivyConf.SESSION_TIMEOUT, "100ms")
+ val manager = new SessionManager[MockSession, RecoveryMetadata](
+ livyConf,
+ { _ => assert(false).asInstanceOf[MockSession] },
+ mock[SessionStore],
+ "test",
+ Some(Seq.empty))
+ val session = manager.register(new MockSession(manager.nextId(), null, livyConf))
+ manager.get(session.id).isDefined should be(true)
+ eventually(timeout(5 seconds), interval(100 millis)) {
+ Await.result(manager.collectGarbage(), Duration.Inf)
+ manager.get(session.id) should be(None)
+ }
+ }
+
+ it("batch session should not be gc-ed until application is finished") {
+ val sessionId = 24
+ val session = mock[BatchSession]
+ when(session.id).thenReturn(sessionId)
+ when(session.stop()).thenReturn(Future {})
+ when(session.lastActivity).thenReturn(System.nanoTime())
+
+ val conf = new LivyConf().set(LivyConf.SESSION_STATE_RETAIN_TIME, "1s")
+ val sm = new BatchSessionManager(conf, mock[SessionStore], Some(Seq(session)))
+ testSessionGC(session, sm)
+ }
+
+ it("interactive session should not gc-ed if session timeout check is off") {
+ val sessionId = 24
+ val session = mock[InteractiveSession]
+ when(session.id).thenReturn(sessionId)
+ when(session.stop()).thenReturn(Future {})
+ when(session.lastActivity).thenReturn(System.nanoTime())
+
+ val conf = new LivyConf().set(LivyConf.SESSION_TIMEOUT_CHECK, false)
+ .set(LivyConf.SESSION_STATE_RETAIN_TIME, "1s")
+ val sm = new InteractiveSessionManager(conf, mock[SessionStore], Some(Seq(session)))
+ testSessionGC(session, sm)
+ }
+
+ def testSessionGC(session: Session, sm: SessionManager[_, _]): Unit = {
+
+ def changeStateAndCheck(s: SessionState)(fn: SessionManager[_, _] => Unit): Unit = {
+ doReturn(s).when(session).state
+ Await.result(sm.collectGarbage(), Duration.Inf)
+ fn(sm)
+ }
+
+ // Batch session should not be gc-ed when alive
+ for (s <- Seq(SessionState.Running(),
+ SessionState.Idle(),
+ SessionState.Recovering(),
+ SessionState.NotStarted(),
+ SessionState.Busy(),
+ SessionState.ShuttingDown())) {
+ changeStateAndCheck(s) { sm => sm.get(session.id) should be (Some(session)) }
+ }
+
+ // Stopped session should be gc-ed after retained timeout
+ for (s <- Seq(SessionState.Error(),
+ SessionState.Success(),
+ SessionState.Dead())) {
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ changeStateAndCheck(s) { sm => sm.get(session.id) should be (None) }
+ }
+ }
+ }
+ }
+
+ describe("BatchSessionManager") {
+ implicit def executor: ExecutionContext = ExecutionContext.global
+
+ def makeMetadata(id: Int, appTag: String): BatchRecoveryMetadata = {
+ BatchRecoveryMetadata(id, None, appTag, null, None)
+ }
+
+ def mockSession(id: Int): BatchSession = {
+ val session = mock[BatchSession]
+ when(session.id).thenReturn(id)
+ when(session.stop()).thenReturn(Future {})
+ when(session.lastActivity).thenReturn(System.nanoTime())
+
+ session
+ }
+
+ it("should not fail if state store is empty") {
+ val conf = new LivyConf()
+
+ val sessionStore = mock[SessionStore]
+ when(sessionStore.getAllSessions[BatchRecoveryMetadata]("batch"))
+ .thenReturn(Seq.empty)
+
+ val sm = new BatchSessionManager(conf, sessionStore)
+ sm.nextId() shouldBe 0
+ }
+
+ it("should recover sessions from state store") {
+ val conf = new LivyConf()
+ conf.set(LivyConf.LIVY_SPARK_MASTER.key, "yarn-cluster")
+
+ val sessionType = "batch"
+ val nextId = 99
+
+ val validMetadata = List(makeMetadata(0, "t1"), makeMetadata(77, "t2")).map(Try(_))
+ val invalidMetadata = List(Failure(new Exception("Fake invalid metadata")))
+ val sessionStore = mock[SessionStore]
+ when(sessionStore.getNextSessionId(sessionType)).thenReturn(nextId)
+ when(sessionStore.getAllSessions[BatchRecoveryMetadata](sessionType))
+ .thenReturn(validMetadata ++ invalidMetadata)
+
+ val sm = new BatchSessionManager(conf, sessionStore)
+ sm.nextId() shouldBe nextId
+ validMetadata.foreach { m =>
+ sm.get(m.get.id) shouldBe defined
+ }
+ sm.size shouldBe validMetadata.size
+ }
+
+ it("should delete sessions from state store") {
+ val conf = new LivyConf()
+
+ val sessionType = "batch"
+ val sessionId = 24
+ val sessionStore = mock[SessionStore]
+ val session = mockSession(sessionId)
+
+ val sm = new BatchSessionManager(conf, sessionStore, Some(Seq(session)))
+ sm.get(sessionId) shouldBe defined
+
+ Await.ready(sm.delete(sessionId).get, 30 seconds)
+
+ verify(sessionStore).remove(sessionType, sessionId)
+ sm.get(sessionId) shouldBe None
+ }
+
+ it("should delete sessions on shutdown when recovery is off") {
+ val conf = new LivyConf()
+ val sessionId = 24
+ val sessionStore = mock[SessionStore]
+ val session = mockSession(sessionId)
+
+ val sm = new BatchSessionManager(conf, sessionStore, Some(Seq(session)))
+ sm.get(sessionId) shouldBe defined
+ sm.shutdown()
+
+ verify(session).stop()
+ }
+
+ it("should not delete sessions on shutdown with recovery is on") {
+ val conf = new LivyConf()
+ conf.set(LivyConf.RECOVERY_MODE, SessionManager.SESSION_RECOVERY_MODE_RECOVERY)
+
+ val sessionId = 24
+ val sessionStore = mock[SessionStore]
+ val session = mockSession(sessionId)
+
+ val sm = new BatchSessionManager(conf, sessionStore, Some(Seq(session)))
+ sm.get(sessionId) shouldBe defined
+ sm.shutdown()
+
+ verify(session, never).stop()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
new file mode 100644
index 0000000..a7bfaaa
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.sessions
+
+import java.net.URI
+
+import org.scalatest.FunSuite
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+
+class SessionSpec extends FunSuite with LivyBaseUnitTestSuite {
+
+ test("use default fs in paths") {
+ val conf = new LivyConf(false)
+ conf.hadoopConf.set("fs.defaultFS", "dummy:///")
+
+ val uris = Seq("http://example.com/foo", "hdfs:/bar", "/baz")
+ val expected = Seq(uris(0), uris(1), "dummy:///baz")
+ assert(Session.resolveURIs(uris, conf) === expected)
+
+ intercept[IllegalArgumentException] {
+ Session.resolveURI(new URI("relative_path"), conf)
+ }
+ }
+
+ test("local fs whitelist") {
+ val conf = new LivyConf(false)
+ conf.set(LivyConf.LOCAL_FS_WHITELIST, "/allowed/,/also_allowed")
+
+ Seq("/allowed/file", "/also_allowed/file").foreach { path =>
+ assert(Session.resolveURI(new URI(path), conf) === new URI("file://" + path))
+ }
+
+ Seq("/not_allowed", "/allowed_not_really").foreach { path =>
+ intercept[IllegalArgumentException] {
+ Session.resolveURI(new URI(path), conf)
+ }
+ }
+ }
+
+ test("conf validation and preparation") {
+ val conf = new LivyConf(false)
+ conf.hadoopConf.set("fs.defaultFS", "dummy:///")
+ conf.set(LivyConf.LOCAL_FS_WHITELIST, "/allowed")
+
+ // Test baseline.
+ assert(Session.prepareConf(Map(), Nil, Nil, Nil, Nil, conf) === Map("spark.master" -> "local"))
+
+ // Test validations.
+ intercept[IllegalArgumentException] {
+ Session.prepareConf(Map("spark.do_not_set" -> "1"), Nil, Nil, Nil, Nil, conf)
+ }
+ conf.sparkFileLists.foreach { key =>
+ intercept[IllegalArgumentException] {
+ Session.prepareConf(Map(key -> "file:/not_allowed"), Nil, Nil, Nil, Nil, conf)
+ }
+ }
+ intercept[IllegalArgumentException] {
+ Session.prepareConf(Map(), Seq("file:/not_allowed"), Nil, Nil, Nil, conf)
+ }
+ intercept[IllegalArgumentException] {
+ Session.prepareConf(Map(), Nil, Seq("file:/not_allowed"), Nil, Nil, conf)
+ }
+ intercept[IllegalArgumentException] {
+ Session.prepareConf(Map(), Nil, Nil, Seq("file:/not_allowed"), Nil, conf)
+ }
+ intercept[IllegalArgumentException] {
+ Session.prepareConf(Map(), Nil, Nil, Nil, Seq("file:/not_allowed"), conf)
+ }
+
+ // Test that file lists are merged and resolved.
+ val base = "/file1.txt"
+ val other = Seq("/file2.txt")
+ val expected = Some(Seq("dummy://" + other(0), "dummy://" + base).mkString(","))
+
+ val userLists = Seq(LivyConf.SPARK_JARS, LivyConf.SPARK_FILES, LivyConf.SPARK_ARCHIVES,
+ LivyConf.SPARK_PY_FILES)
+ val baseConf = userLists.map { key => (key -> base) }.toMap
+ val result = Session.prepareConf(baseConf, other, other, other, other, conf)
+ userLists.foreach { key => assert(result.get(key) === expected) }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/utils/LivySparkUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/utils/LivySparkUtilsSuite.scala b/server/src/test/scala/org/apache/livy/utils/LivySparkUtilsSuite.scala
new file mode 100644
index 0000000..eb5f30a
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/utils/LivySparkUtilsSuite.scala
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.utils
+
+import org.scalatest.FunSuite
+import org.scalatest.Matchers
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+import org.apache.livy.LivyConf._
+import org.apache.livy.server.LivyServer
+
+class LivySparkUtilsSuite extends FunSuite with Matchers with LivyBaseUnitTestSuite {
+
+ import LivySparkUtils._
+
+ private val livyConf = new LivyConf()
+ private val livyConf210 = new LivyConf()
+ livyConf210.set(LIVY_SPARK_SCALA_VERSION, "2.10.6")
+
+ private val livyConf211 = new LivyConf()
+ livyConf211.set(LIVY_SPARK_SCALA_VERSION, "2.11.1")
+
+ test("check for SPARK_HOME") {
+ testSparkHome(livyConf)
+ }
+
+ test("check spark-submit version") {
+ testSparkSubmit(livyConf)
+ }
+
+ test("should support Spark 1.6") {
+ testSparkVersion("1.6.0")
+ testSparkVersion("1.6.1")
+ testSparkVersion("1.6.1-SNAPSHOT")
+ testSparkVersion("1.6.2")
+ testSparkVersion("1.6")
+ testSparkVersion("1.6.3.2.5.0-12")
+ }
+
+ test("should support Spark 2.0.x") {
+ testSparkVersion("2.0.0")
+ testSparkVersion("2.0.1")
+ testSparkVersion("2.0.2")
+ testSparkVersion("2.0.3-SNAPSHOT")
+ testSparkVersion("2.0.0.2.5.1.0-56") // LIVY-229
+ testSparkVersion("2.0")
+ testSparkVersion("2.1.0")
+ testSparkVersion("2.1.1")
+ }
+
+ test("should not support Spark older than 1.6") {
+ intercept[IllegalArgumentException] { testSparkVersion("1.4.0") }
+ intercept[IllegalArgumentException] { testSparkVersion("1.5.0") }
+ intercept[IllegalArgumentException] { testSparkVersion("1.5.1") }
+ intercept[IllegalArgumentException] { testSparkVersion("1.5.2") }
+ intercept[IllegalArgumentException] { testSparkVersion("1.5.0-cdh5.6.1") }
+ }
+
+ test("should fail on bad version") {
+ intercept[IllegalArgumentException] { testSparkVersion("not a version") }
+ }
+
+ test("should error out if recovery is turned on but master isn't yarn") {
+ val livyConf = new LivyConf()
+ livyConf.set(LivyConf.LIVY_SPARK_MASTER, "local")
+ livyConf.set(LivyConf.RECOVERY_MODE, "recovery")
+ val s = new LivyServer()
+ intercept[IllegalArgumentException] { s.testRecovery(livyConf) }
+ }
+
+ test("formatScalaVersion() should format Scala version") {
+ formatScalaVersion("2.10.8") shouldBe "2.10"
+ formatScalaVersion("2.11.4") shouldBe "2.11"
+ formatScalaVersion("2.10") shouldBe "2.10"
+ formatScalaVersion("2.10.x.x.x.x") shouldBe "2.10"
+
+ // Throw exception for bad Scala version.
+ intercept[IllegalArgumentException] { formatScalaVersion("") }
+ intercept[IllegalArgumentException] { formatScalaVersion("xxx") }
+ }
+
+ test("defaultSparkScalaVersion() should return default Scala version") {
+ defaultSparkScalaVersion(formatSparkVersion("1.6.0")) shouldBe "2.10"
+ defaultSparkScalaVersion(formatSparkVersion("1.6.1")) shouldBe "2.10"
+ defaultSparkScalaVersion(formatSparkVersion("1.6.2")) shouldBe "2.10"
+ defaultSparkScalaVersion(formatSparkVersion("2.0.0")) shouldBe "2.11"
+ defaultSparkScalaVersion(formatSparkVersion("2.0.1")) shouldBe "2.11"
+
+ // Throw exception for unsupported Spark version.
+ intercept[IllegalArgumentException] { defaultSparkScalaVersion(formatSparkVersion("1.5.0")) }
+ }
+
+ test("sparkScalaVersion() should use spark-submit detected Scala version.") {
+ sparkScalaVersion(formatSparkVersion("2.0.1"), Some("2.10"), livyConf) shouldBe "2.10"
+ sparkScalaVersion(formatSparkVersion("1.6.0"), Some("2.11"), livyConf) shouldBe "2.11"
+ }
+
+ test("sparkScalaVersion() should throw if configured and detected Scala version mismatch.") {
+ intercept[IllegalArgumentException] {
+ sparkScalaVersion(formatSparkVersion("2.0.1"), Some("2.11"), livyConf210)
+ }
+ intercept[IllegalArgumentException] {
+ sparkScalaVersion(formatSparkVersion("1.6.1"), Some("2.10"), livyConf211)
+ }
+ }
+
+ test("sparkScalaVersion() should use configured Scala version if spark-submit doesn't tell.") {
+ sparkScalaVersion(formatSparkVersion("1.6.0"), None, livyConf210) shouldBe "2.10"
+ sparkScalaVersion(formatSparkVersion("1.6.2"), None, livyConf210) shouldBe "2.10"
+ sparkScalaVersion(formatSparkVersion("2.0.0"), None, livyConf210) shouldBe "2.10"
+ sparkScalaVersion(formatSparkVersion("2.0.1"), None, livyConf210) shouldBe "2.10"
+ sparkScalaVersion(formatSparkVersion("1.6.0"), None, livyConf211) shouldBe "2.11"
+ sparkScalaVersion(formatSparkVersion("1.6.2"), None, livyConf211) shouldBe "2.11"
+ sparkScalaVersion(formatSparkVersion("2.0.0"), None, livyConf211) shouldBe "2.11"
+ sparkScalaVersion(formatSparkVersion("2.0.1"), None, livyConf211) shouldBe "2.11"
+ }
+
+ test("sparkScalaVersion() should use default Spark Scala version.") {
+ sparkScalaVersion(formatSparkVersion("1.6.0"), None, livyConf) shouldBe "2.10"
+ sparkScalaVersion(formatSparkVersion("1.6.2"), None, livyConf) shouldBe "2.10"
+ sparkScalaVersion(formatSparkVersion("2.0.0"), None, livyConf) shouldBe "2.11"
+ sparkScalaVersion(formatSparkVersion("2.0.1"), None, livyConf) shouldBe "2.11"
+ sparkScalaVersion(formatSparkVersion("2.1.0"), None, livyConf) shouldBe "2.11"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala b/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala
new file mode 100644
index 0000000..b3c50da
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.utils
+
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.apache.hadoop.yarn.api.records._
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus.UNDEFINED
+import org.apache.hadoop.yarn.api.records.YarnApplicationState._
+import org.apache.hadoop.yarn.client.api.YarnClient
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException
+import org.apache.hadoop.yarn.util.ConverterUtils
+import org.mockito.Mockito._
+import org.mockito.invocation.InvocationOnMock
+import org.mockito.stubbing.Answer
+import org.scalatest.FunSpec
+import org.scalatest.mock.MockitoSugar.mock
+
+import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
+import org.apache.livy.utils.SparkApp._
+
+class SparkYarnAppSpec extends FunSpec with LivyBaseUnitTestSuite {
+ private def cleanupThread(t: Thread)(f: => Unit) = {
+ try { f } finally { t.interrupt() }
+ }
+
+ private def mockSleep(ms: Long) = {
+ Thread.`yield`()
+ }
+
+ describe("SparkYarnApp") {
+ val TEST_TIMEOUT = 30 seconds
+ val appId = ConverterUtils.toApplicationId("application_1467912463905_0021")
+ val appIdOption = Some(appId.toString)
+ val appTag = "fakeTag"
+ val livyConf = new LivyConf()
+ livyConf.set(LivyConf.YARN_APP_LOOKUP_TIMEOUT, "30s")
+
+ it("should poll YARN state and terminate") {
+ Clock.withSleepMethod(mockSleep) {
+ val mockYarnClient = mock[YarnClient]
+ val mockAppListener = mock[SparkAppListener]
+
+ val mockAppReport = mock[ApplicationReport]
+ when(mockAppReport.getApplicationId).thenReturn(appId)
+ when(mockAppReport.getFinalApplicationStatus).thenReturn(FinalApplicationStatus.SUCCEEDED)
+ // Simulate YARN app state progression.
+ when(mockAppReport.getYarnApplicationState).thenAnswer(new Answer[YarnApplicationState]() {
+ private var stateSeq = List(ACCEPTED, RUNNING, FINISHED)
+
+ override def answer(invocation: InvocationOnMock): YarnApplicationState = {
+ val currentState = stateSeq.head
+ if (stateSeq.tail.nonEmpty) {
+ stateSeq = stateSeq.tail
+ }
+ currentState
+ }
+ })
+ when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
+
+ val app = new SparkYarnApp(
+ appTag,
+ appIdOption,
+ None,
+ Some(mockAppListener),
+ livyConf,
+ mockYarnClient)
+ cleanupThread(app.yarnAppMonitorThread) {
+ app.yarnAppMonitorThread.join(TEST_TIMEOUT.toMillis)
+ assert(!app.yarnAppMonitorThread.isAlive,
+ "YarnAppMonitorThread should terminate after YARN app is finished.")
+ verify(mockYarnClient, atLeast(1)).getApplicationReport(appId)
+ verify(mockAppListener).stateChanged(State.STARTING, State.RUNNING)
+ verify(mockAppListener).stateChanged(State.RUNNING, State.FINISHED)
+ }
+ }
+ }
+
+ it("should kill yarn app") {
+ Clock.withSleepMethod(mockSleep) {
+ val diag = "DIAG"
+ val mockYarnClient = mock[YarnClient]
+
+ val mockAppReport = mock[ApplicationReport]
+ when(mockAppReport.getApplicationId).thenReturn(appId)
+ when(mockAppReport.getDiagnostics).thenReturn(diag)
+ when(mockAppReport.getFinalApplicationStatus).thenReturn(FinalApplicationStatus.SUCCEEDED)
+
+ var appKilled = false
+ when(mockAppReport.getYarnApplicationState).thenAnswer(new Answer[YarnApplicationState]() {
+ override def answer(invocation: InvocationOnMock): YarnApplicationState = {
+ if (!appKilled) {
+ RUNNING
+ } else {
+ KILLED
+ }
+ }
+ })
+ when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
+
+ val app = new SparkYarnApp(appTag, appIdOption, None, None, livyConf, mockYarnClient)
+ cleanupThread(app.yarnAppMonitorThread) {
+ app.kill()
+ appKilled = true
+
+ app.yarnAppMonitorThread.join(TEST_TIMEOUT.toMillis)
+ assert(!app.yarnAppMonitorThread.isAlive,
+ "YarnAppMonitorThread should terminate after YARN app is finished.")
+ verify(mockYarnClient, atLeast(1)).getApplicationReport(appId)
+ verify(mockYarnClient).killApplication(appId)
+ assert(app.log().mkString.contains(diag))
+ }
+ }
+ }
+
+ it("should return spark-submit log") {
+ Clock.withSleepMethod(mockSleep) {
+ val mockYarnClient = mock[YarnClient]
+ val mockSparkSubmit = mock[LineBufferedProcess]
+ val sparkSubmitInfoLog = IndexedSeq("SPARK-SUBMIT", "LOG")
+ val sparkSubmitErrorLog = IndexedSeq("SPARK-SUBMIT", "error log")
+ val sparkSubmitLog = ("stdout: " +: sparkSubmitInfoLog) ++
+ ("\nstderr: " +: sparkSubmitErrorLog) :+ "\nYARN Diagnostics: "
+ when(mockSparkSubmit.inputLines).thenReturn(sparkSubmitInfoLog)
+ when(mockSparkSubmit.errorLines).thenReturn(sparkSubmitErrorLog)
+ val waitForCalledLatch = new CountDownLatch(1)
+ when(mockSparkSubmit.waitFor()).thenAnswer(new Answer[Int]() {
+ override def answer(invocation: InvocationOnMock): Int = {
+ waitForCalledLatch.countDown()
+ 0
+ }
+ })
+
+ val mockAppReport = mock[ApplicationReport]
+ when(mockAppReport.getApplicationId).thenReturn(appId)
+ when(mockAppReport.getYarnApplicationState).thenReturn(YarnApplicationState.FINISHED)
+ when(mockAppReport.getDiagnostics).thenReturn(null)
+ when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
+
+ val app = new SparkYarnApp(
+ appTag,
+ appIdOption,
+ Some(mockSparkSubmit),
+ None,
+ livyConf,
+ mockYarnClient)
+ cleanupThread(app.yarnAppMonitorThread) {
+ waitForCalledLatch.await(TEST_TIMEOUT.toMillis, TimeUnit.MILLISECONDS)
+ assert(app.log() == sparkSubmitLog, "Expect spark-submit log")
+ }
+ }
+ }
+
+ it("can kill spark-submit while it's running") {
+ Clock.withSleepMethod(mockSleep) {
+ val livyConf = new LivyConf()
+ livyConf.set(LivyConf.YARN_APP_LOOKUP_TIMEOUT, "0")
+
+ val mockYarnClient = mock[YarnClient]
+ val mockSparkSubmit = mock[LineBufferedProcess]
+
+ val sparkSubmitRunningLatch = new CountDownLatch(1)
+ // Simulate a running spark-submit
+ when(mockSparkSubmit.waitFor()).thenAnswer(new Answer[Int]() {
+ override def answer(invocation: InvocationOnMock): Int = {
+ sparkSubmitRunningLatch.await()
+ 0
+ }
+ })
+
+ val app = new SparkYarnApp(
+ appTag,
+ appIdOption,
+ Some(mockSparkSubmit),
+ None,
+ livyConf,
+ mockYarnClient)
+ cleanupThread(app.yarnAppMonitorThread) {
+ app.kill()
+ verify(mockSparkSubmit, times(1)).destroy()
+ sparkSubmitRunningLatch.countDown()
+ }
+ }
+ }
+
+ it("should map YARN state to SparkApp.State correctly") {
+ val app = new SparkYarnApp(appTag, appIdOption, None, None, livyConf)
+ cleanupThread(app.yarnAppMonitorThread) {
+ assert(app.mapYarnState(appId, NEW, UNDEFINED) == State.STARTING)
+ assert(app.mapYarnState(appId, NEW_SAVING, UNDEFINED) == State.STARTING)
+ assert(app.mapYarnState(appId, SUBMITTED, UNDEFINED) == State.STARTING)
+ assert(app.mapYarnState(appId, ACCEPTED, UNDEFINED) == State.STARTING)
+ assert(app.mapYarnState(appId, RUNNING, UNDEFINED) == State.RUNNING)
+ assert(
+ app.mapYarnState(appId, FINISHED, FinalApplicationStatus.SUCCEEDED) == State.FINISHED)
+ assert(app.mapYarnState(appId, FINISHED, FinalApplicationStatus.FAILED) == State.FAILED)
+ assert(app.mapYarnState(appId, FINISHED, FinalApplicationStatus.KILLED) == State.KILLED)
+ assert(app.mapYarnState(appId, FINISHED, UNDEFINED) == State.FAILED)
+ assert(app.mapYarnState(appId, FAILED, UNDEFINED) == State.FAILED)
+ assert(app.mapYarnState(appId, KILLED, UNDEFINED) == State.KILLED)
+ }
+ }
+
+ it("should expose driver log url and Spark UI url") {
+ Clock.withSleepMethod(mockSleep) {
+ val mockYarnClient = mock[YarnClient]
+ val driverLogUrl = "DRIVER LOG URL"
+ val sparkUiUrl = "SPARK UI URL"
+
+ val mockApplicationAttemptId = mock[ApplicationAttemptId]
+ val mockAppReport = mock[ApplicationReport]
+ when(mockAppReport.getApplicationId).thenReturn(appId)
+ when(mockAppReport.getFinalApplicationStatus).thenReturn(FinalApplicationStatus.SUCCEEDED)
+ when(mockAppReport.getTrackingUrl).thenReturn(sparkUiUrl)
+ when(mockAppReport.getCurrentApplicationAttemptId).thenReturn(mockApplicationAttemptId)
+ var done = false
+ when(mockAppReport.getYarnApplicationState).thenAnswer(new Answer[YarnApplicationState]() {
+ override def answer(invocation: InvocationOnMock): YarnApplicationState = {
+ if (!done) {
+ RUNNING
+ } else {
+ FINISHED
+ }
+ }
+ })
+ when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
+
+ val mockAttemptReport = mock[ApplicationAttemptReport]
+ val mockContainerId = mock[ContainerId]
+ when(mockAttemptReport.getAMContainerId).thenReturn(mockContainerId)
+ when(mockYarnClient.getApplicationAttemptReport(mockApplicationAttemptId))
+ .thenReturn(mockAttemptReport)
+
+ val mockContainerReport = mock[ContainerReport]
+ when(mockYarnClient.getContainerReport(mockContainerId)).thenReturn(mockContainerReport)
+
+ // Block test until getLogUrl is called 10 times.
+ val getLogUrlCountDown = new CountDownLatch(10)
+ when(mockContainerReport.getLogUrl).thenAnswer(new Answer[String] {
+ override def answer(invocation: InvocationOnMock): String = {
+ getLogUrlCountDown.countDown()
+ driverLogUrl
+ }
+ })
+
+ val mockListener = mock[SparkAppListener]
+
+ val app = new SparkYarnApp(
+ appTag, appIdOption, None, Some(mockListener), livyConf, mockYarnClient)
+ cleanupThread(app.yarnAppMonitorThread) {
+ getLogUrlCountDown.await(TEST_TIMEOUT.length, TEST_TIMEOUT.unit)
+ done = true
+
+ app.yarnAppMonitorThread.join(TEST_TIMEOUT.toMillis)
+ assert(!app.yarnAppMonitorThread.isAlive,
+ "YarnAppMonitorThread should terminate after YARN app is finished.")
+
+ verify(mockYarnClient, atLeast(1)).getApplicationReport(appId)
+ verify(mockAppReport, atLeast(1)).getTrackingUrl()
+ verify(mockContainerReport, atLeast(1)).getLogUrl()
+ verify(mockListener).appIdKnown(appId.toString)
+ verify(mockListener).infoChanged(AppInfo(Some(driverLogUrl), Some(sparkUiUrl)))
+ }
+ }
+ }
+
+ it("should not die on YARN-4411") {
+ Clock.withSleepMethod(mockSleep) {
+ val mockYarnClient = mock[YarnClient]
+
+ // Block test until getApplicationReport is called 10 times.
+ val pollCountDown = new CountDownLatch(10)
+ when(mockYarnClient.getApplicationReport(appId)).thenAnswer(new Answer[ApplicationReport] {
+ override def answer(invocation: InvocationOnMock): ApplicationReport = {
+ pollCountDown.countDown()
+ throw new IllegalArgumentException("No enum constant " +
+ "org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState.FINAL_SAVING")
+ }
+ })
+
+ val app = new SparkYarnApp(appTag, appIdOption, None, None, livyConf, mockYarnClient)
+ cleanupThread(app.yarnAppMonitorThread) {
+ pollCountDown.await(TEST_TIMEOUT.length, TEST_TIMEOUT.unit)
+ assert(app.state == SparkApp.State.STARTING)
+
+ app.state = SparkApp.State.FINISHED
+ app.yarnAppMonitorThread.join(TEST_TIMEOUT.toMillis)
+ }
+ }
+ }
+
+ it("should not die on ApplicationAttemptNotFoundException") {
+ Clock.withSleepMethod(mockSleep) {
+ val mockYarnClient = mock[YarnClient]
+ val mockAppReport = mock[ApplicationReport]
+ val mockApplicationAttemptId = mock[ApplicationAttemptId]
+ var done = false
+
+ when(mockAppReport.getApplicationId).thenReturn(appId)
+ when(mockAppReport.getYarnApplicationState).thenAnswer(
+ new Answer[YarnApplicationState]() {
+ override def answer(invocation: InvocationOnMock): YarnApplicationState = {
+ if (done) {
+ FINISHED
+ } else {
+ RUNNING
+ }
+ }
+ })
+ when(mockAppReport.getFinalApplicationStatus).thenReturn(FinalApplicationStatus.SUCCEEDED)
+ when(mockAppReport.getCurrentApplicationAttemptId).thenReturn(mockApplicationAttemptId)
+ when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
+
+ // Block test until getApplicationReport is called 10 times.
+ val pollCountDown = new CountDownLatch(10)
+ when(mockYarnClient.getApplicationAttemptReport(mockApplicationAttemptId)).thenAnswer(
+ new Answer[ApplicationReport] {
+ override def answer(invocation: InvocationOnMock): ApplicationReport = {
+ pollCountDown.countDown()
+ throw new ApplicationAttemptNotFoundException("unit test")
+ }
+ })
+
+ val app = new SparkYarnApp(appTag, appIdOption, None, None, livyConf, mockYarnClient)
+ cleanupThread(app.yarnAppMonitorThread) {
+ pollCountDown.await(TEST_TIMEOUT.length, TEST_TIMEOUT.unit)
+ assert(app.state == SparkApp.State.RUNNING)
+ done = true
+
+ app.yarnAppMonitorThread.join(TEST_TIMEOUT.toMillis)
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/pom.xml
----------------------------------------------------------------------
diff --git a/test-lib/pom.xml b/test-lib/pom.xml
index 131e0e6..c380764 100644
--- a/test-lib/pom.xml
+++ b/test-lib/pom.xml
@@ -18,14 +18,14 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
</parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-test-lib</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -34,7 +34,7 @@
<dependencies>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-api</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/com/cloudera/livy/test/apps/FailingApp.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/com/cloudera/livy/test/apps/FailingApp.java b/test-lib/src/main/java/com/cloudera/livy/test/apps/FailingApp.java
deleted file mode 100644
index 51ba795..0000000
--- a/test-lib/src/main/java/com/cloudera/livy/test/apps/FailingApp.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.apps;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public class FailingApp {
-
- public static void main(String[] args) throws Exception {
- if (args.length != 1) {
- throw new IllegalArgumentException("Missing output path.");
- }
- String output = args[0];
-
- FileSystem fs = FileSystem.get(new Configuration());
- Path out = new Path(output);
- fs.create(out).close();
-
- throw new IllegalStateException("This app always fails.");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/com/cloudera/livy/test/apps/SimpleSparkApp.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/com/cloudera/livy/test/apps/SimpleSparkApp.java b/test-lib/src/main/java/com/cloudera/livy/test/apps/SimpleSparkApp.java
deleted file mode 100644
index 922066e..0000000
--- a/test-lib/src/main/java/com/cloudera/livy/test/apps/SimpleSparkApp.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.apps;
-
-import java.util.Arrays;
-import java.util.List;
-
-import scala.Tuple2;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.PairFunction;
-
-public class SimpleSparkApp {
-
- public static void main(String[] args) throws Exception {
- if (args.length < 1 || args.length > 2) {
- throw new IllegalArgumentException(
- "Invalid arguments. <output path> [exit after output=true]>");
- }
-
- String output = args[0];
- Boolean exitAfterOutput = true;
- if (args.length == 2) {
- exitAfterOutput = Boolean.parseBoolean(args[1]);
- }
-
- JavaSparkContext sc = new JavaSparkContext();
- try {
- List<String> data = Arrays.asList("the", "quick", "brown", "fox", "jumped", "over", "the",
- "lazy", "dog");
-
- JavaPairRDD<String, Integer> rdd = sc.parallelize(data, 3)
- .mapToPair(new Counter());
- rdd.saveAsTextFile(output);
-
- if (!exitAfterOutput) {
- while (true) {
- Thread.sleep(60 * 60 * 1000);
- }
- }
- } finally {
- sc.close();
- }
- }
-
- private static class Counter implements PairFunction<String, String, Integer> {
-
- @Override
- public Tuple2<String, Integer> call(String s) throws Exception {
- return new Tuple2<>(s, s.length());
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/com/cloudera/livy/test/jobs/Echo.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/com/cloudera/livy/test/jobs/Echo.java b/test-lib/src/main/java/com/cloudera/livy/test/jobs/Echo.java
deleted file mode 100644
index 7ea3aa7..0000000
--- a/test-lib/src/main/java/com/cloudera/livy/test/jobs/Echo.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class Echo<T> implements Job<T> {
-
- private final T value;
-
- public Echo(T value) {
- this.value = value;
- }
-
- @Override
- public T call(JobContext jc) {
- return value;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/com/cloudera/livy/test/jobs/Failure.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/com/cloudera/livy/test/jobs/Failure.java b/test-lib/src/main/java/com/cloudera/livy/test/jobs/Failure.java
deleted file mode 100644
index 9232347..0000000
--- a/test-lib/src/main/java/com/cloudera/livy/test/jobs/Failure.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class Failure implements Job<Void> {
-
- @Override
- public Void call(JobContext jc) {
- throw new JobFailureException();
- }
-
- public static class JobFailureException extends RuntimeException {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/com/cloudera/livy/test/jobs/FileReader.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/com/cloudera/livy/test/jobs/FileReader.java b/test-lib/src/main/java/com/cloudera/livy/test/jobs/FileReader.java
deleted file mode 100644
index 3bca2f3..0000000
--- a/test-lib/src/main/java/com/cloudera/livy/test/jobs/FileReader.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import org.apache.spark.SparkFiles;
-import org.apache.spark.api.java.function.Function;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class FileReader implements Job<String> {
-
- private final boolean isResource;
- private final String fileName;
-
- public FileReader(String fileName, boolean isResource) {
- this.fileName = fileName;
- this.isResource = isResource;
- }
-
- @Override
- public String call(JobContext jc) {
- return jc.sc().parallelize(Arrays.asList(1)).map(new Reader()).collect().get(0);
- }
-
- private class Reader implements Function<Integer, String> {
-
- @Override
- public String call(Integer i) throws Exception {
- InputStream in;
- if (isResource) {
- ClassLoader ccl = Thread.currentThread().getContextClassLoader();
- in = ccl.getResourceAsStream(fileName);
- if (in == null) {
- throw new IOException("Resource not found: " + fileName);
- }
- } else {
- in = new FileInputStream(SparkFiles.get(fileName));
- }
- try {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- byte[] buf = new byte[1024];
- int read;
- while ((read = in.read(buf)) >= 0) {
- out.write(buf, 0, read);
- }
- byte[] bytes = out.toByteArray();
- return new String(bytes, 0, bytes.length, UTF_8);
- } finally {
- in.close();
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/com/cloudera/livy/test/jobs/GetCurrentUser.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/com/cloudera/livy/test/jobs/GetCurrentUser.java b/test-lib/src/main/java/com/cloudera/livy/test/jobs/GetCurrentUser.java
deleted file mode 100644
index 5919896..0000000
--- a/test-lib/src/main/java/com/cloudera/livy/test/jobs/GetCurrentUser.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs;
-
-import org.apache.hadoop.security.UserGroupInformation;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class GetCurrentUser implements Job<String> {
-
- @Override
- public String call(JobContext jc) throws Exception {
- return UserGroupInformation.getCurrentUser().getUserName();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/com/cloudera/livy/test/jobs/SQLGetTweets.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/com/cloudera/livy/test/jobs/SQLGetTweets.java b/test-lib/src/main/java/com/cloudera/livy/test/jobs/SQLGetTweets.java
deleted file mode 100644
index 15aa79b..0000000
--- a/test-lib/src/main/java/com/cloudera/livy/test/jobs/SQLGetTweets.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs;
-
-import java.io.File;
-import java.io.InputStream;
-import java.net.URI;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class SQLGetTweets implements Job<List<String>> {
-
- private final boolean useHiveContext;
-
- public SQLGetTweets(boolean useHiveContext) {
- this.useHiveContext = useHiveContext;
- }
-
- @Override
- public List<String> call(JobContext jc) throws Exception {
- InputStream source = getClass().getResourceAsStream("/testweet.json");
-
- // Save the resource as a file in HDFS (or the local tmp dir when using a local filesystem).
- URI input;
- File local = File.createTempFile("tweets", ".json", jc.getLocalTmpDir());
- Files.copy(source, local.toPath(), StandardCopyOption.REPLACE_EXISTING);
- FileSystem fs = FileSystem.get(jc.sc().sc().hadoopConfiguration());
- if ("file".equals(fs.getUri().getScheme())) {
- input = local.toURI();
- } else {
- String uuid = UUID.randomUUID().toString();
- Path target = new Path("/tmp/" + uuid + "-tweets.json");
- fs.copyFromLocalFile(new Path(local.toURI()), target);
- input = target.toUri();
- }
-
- SQLContext sqlctx = useHiveContext ? jc.hivectx() : jc.sqlctx();
- sqlctx.jsonFile(input.toString()).registerTempTable("tweets");
-
- List<String> tweetList = new ArrayList<>();
- Row[] result =
- (Row[])(sqlctx.sql("SELECT text, retweetCount FROM tweets ORDER BY retweetCount LIMIT 10")
- .collect());
- for (Row r : result) {
- tweetList.add(r.toString());
- }
- return tweetList;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/com/cloudera/livy/test/jobs/Sleeper.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/com/cloudera/livy/test/jobs/Sleeper.java b/test-lib/src/main/java/com/cloudera/livy/test/jobs/Sleeper.java
deleted file mode 100644
index a3c33d4..0000000
--- a/test-lib/src/main/java/com/cloudera/livy/test/jobs/Sleeper.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class Sleeper implements Job<Void> {
-
- private final long millis;
-
- public Sleeper(long millis) {
- this.millis = millis;
- }
-
- @Override
- public Void call(JobContext jc) throws Exception {
- Thread.sleep(millis);
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/com/cloudera/livy/test/jobs/SmallCount.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/com/cloudera/livy/test/jobs/SmallCount.java b/test-lib/src/main/java/com/cloudera/livy/test/jobs/SmallCount.java
deleted file mode 100644
index fb74027..0000000
--- a/test-lib/src/main/java/com/cloudera/livy/test/jobs/SmallCount.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class SmallCount implements Job<Long> {
-
- private final int count;
-
- public SmallCount(int count) {
- this.count = count;
- }
-
- @Override
- public Long call(JobContext jc) {
- Random r = new Random();
- int partitions = Math.min(r.nextInt(10) + 1, count);
-
- List<Integer> elements = new ArrayList<>(count);
- for (int i = 0; i < count; i++) {
- elements.add(r.nextInt());
- }
-
- return jc.sc().parallelize(elements, partitions).count();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/com/cloudera/livy/test/jobs/VoidJob.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/com/cloudera/livy/test/jobs/VoidJob.java b/test-lib/src/main/java/com/cloudera/livy/test/jobs/VoidJob.java
deleted file mode 100644
index 0333ccf..0000000
--- a/test-lib/src/main/java/com/cloudera/livy/test/jobs/VoidJob.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.jobs;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-
-public class VoidJob implements Job<Void> {
- @Override
- public Void call(JobContext jc) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/org/apache/livy/test/apps/FailingApp.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/org/apache/livy/test/apps/FailingApp.java b/test-lib/src/main/java/org/apache/livy/test/apps/FailingApp.java
new file mode 100644
index 0000000..6fa4850
--- /dev/null
+++ b/test-lib/src/main/java/org/apache/livy/test/apps/FailingApp.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.apps;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class FailingApp {
+
+ public static void main(String[] args) throws Exception {
+ if (args.length != 1) {
+ throw new IllegalArgumentException("Missing output path.");
+ }
+ String output = args[0];
+
+ FileSystem fs = FileSystem.get(new Configuration());
+ Path out = new Path(output);
+ fs.create(out).close();
+
+ throw new IllegalStateException("This app always fails.");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/org/apache/livy/test/apps/SimpleSparkApp.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/org/apache/livy/test/apps/SimpleSparkApp.java b/test-lib/src/main/java/org/apache/livy/test/apps/SimpleSparkApp.java
new file mode 100644
index 0000000..ef8e1a6
--- /dev/null
+++ b/test-lib/src/main/java/org/apache/livy/test/apps/SimpleSparkApp.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.apps;
+
+import java.util.Arrays;
+import java.util.List;
+
+import scala.Tuple2;
+
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.PairFunction;
+
+public class SimpleSparkApp {
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 1 || args.length > 2) {
+ throw new IllegalArgumentException(
+ "Invalid arguments. <output path> [exit after output=true]>");
+ }
+
+ String output = args[0];
+ Boolean exitAfterOutput = true;
+ if (args.length == 2) {
+ exitAfterOutput = Boolean.parseBoolean(args[1]);
+ }
+
+ JavaSparkContext sc = new JavaSparkContext();
+ try {
+ List<String> data = Arrays.asList("the", "quick", "brown", "fox", "jumped", "over", "the",
+ "lazy", "dog");
+
+ JavaPairRDD<String, Integer> rdd = sc.parallelize(data, 3)
+ .mapToPair(new Counter());
+ rdd.saveAsTextFile(output);
+
+ if (!exitAfterOutput) {
+ while (true) {
+ Thread.sleep(60 * 60 * 1000);
+ }
+ }
+ } finally {
+ sc.close();
+ }
+ }
+
+ private static class Counter implements PairFunction<String, String, Integer> {
+
+ @Override
+ public Tuple2<String, Integer> call(String s) throws Exception {
+ return new Tuple2<>(s, s.length());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/test-lib/src/main/java/org/apache/livy/test/jobs/Echo.java
----------------------------------------------------------------------
diff --git a/test-lib/src/main/java/org/apache/livy/test/jobs/Echo.java b/test-lib/src/main/java/org/apache/livy/test/jobs/Echo.java
new file mode 100644
index 0000000..3c3c834
--- /dev/null
+++ b/test-lib/src/main/java/org/apache/livy/test/jobs/Echo.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.jobs;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+
+public class Echo<T> implements Job<T> {
+
+ private final T value;
+
+ public Echo(T value) {
+ this.value = value;
+ }
+
+ @Override
+ public T call(JobContext jc) {
+ return value;
+ }
+
+}
[15/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery.dataTables.min.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery.dataTables.min.js b/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery.dataTables.min.js
deleted file mode 100644
index dc969ee..0000000
--- a/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery.dataTables.min.js
+++ /dev/null
@@ -1,167 +0,0 @@
-/*!
- DataTables 1.10.15
- ©2008-2017 SpryMedia Ltd - datatables.net/license
-*/
-(function(h){"function"===typeof define&&define.amd?define(["jquery"],function(E){return h(E,window,document)}):"object"===typeof exports?module.exports=function(E,H){E||(E=window);H||(H="undefined"!==typeof window?require("jquery"):require("jquery")(E));return h(H,E,E.document)}:h(jQuery,window,document)})(function(h,E,H,k){function Y(a){var b,c,d={};h.each(a,function(e){if((b=e.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" "))c=e.replace(b[0],b[2].toLowerCase()),
-d[c]=e,"o"===b[1]&&Y(a[e])});a._hungarianMap=d}function J(a,b,c){a._hungarianMap||Y(a);var d;h.each(b,function(e){d=a._hungarianMap[e];if(d!==k&&(c||b[d]===k))"o"===d.charAt(0)?(b[d]||(b[d]={}),h.extend(!0,b[d],b[e]),J(a[d],b[d],c)):b[d]=b[e]})}function Fa(a){var b=m.defaults.oLanguage,c=a.sZeroRecords;!a.sEmptyTable&&(c&&"No data available in table"===b.sEmptyTable)&&F(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&(c&&"Loading..."===b.sLoadingRecords)&&F(a,a,"sZeroRecords","sLoadingRecords");
-a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&fb(a)}function gb(a){A(a,"ordering","bSort");A(a,"orderMulti","bSortMulti");A(a,"orderClasses","bSortClasses");A(a,"orderCellsTop","bSortCellsTop");A(a,"order","aaSorting");A(a,"orderFixed","aaSortingFixed");A(a,"paging","bPaginate");A(a,"pagingType","sPaginationType");A(a,"pageLength","iDisplayLength");A(a,"searching","bFilter");"boolean"===typeof a.sScrollX&&(a.sScrollX=a.sScrollX?"100%":"");"boolean"===typeof a.scrollX&&(a.scrollX=
-a.scrollX?"100%":"");if(a=a.aoSearchCols)for(var b=0,c=a.length;b<c;b++)a[b]&&J(m.models.oSearch,a[b])}function hb(a){A(a,"orderable","bSortable");A(a,"orderData","aDataSort");A(a,"orderSequence","asSorting");A(a,"orderDataType","sortDataType");var b=a.aDataSort;"number"===typeof b&&!h.isArray(b)&&(a.aDataSort=[b])}function ib(a){if(!m.__browser){var b={};m.__browser=b;var c=h("<div/>").css({position:"fixed",top:0,left:-1*h(E).scrollLeft(),height:1,width:1,overflow:"hidden"}).append(h("<div/>").css({position:"absolute",
-top:1,left:1,width:100,overflow:"scroll"}).append(h("<div/>").css({width:"100%",height:10}))).appendTo("body"),d=c.children(),e=d.children();b.barWidth=d[0].offsetWidth-d[0].clientWidth;b.bScrollOversize=100===e[0].offsetWidth&&100!==d[0].clientWidth;b.bScrollbarLeft=1!==Math.round(e.offset().left);b.bBounding=c[0].getBoundingClientRect().width?!0:!1;c.remove()}h.extend(a.oBrowser,m.__browser);a.oScroll.iBarWidth=m.__browser.barWidth}function jb(a,b,c,d,e,f){var g,j=!1;c!==k&&(g=c,j=!0);for(;d!==
-e;)a.hasOwnProperty(d)&&(g=j?b(g,a[d],d,a):a[d],j=!0,d+=f);return g}function Ga(a,b){var c=m.defaults.column,d=a.aoColumns.length,c=h.extend({},m.models.oColumn,c,{nTh:b?b:H.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.mData:d,idx:d});a.aoColumns.push(c);c=a.aoPreSearchCols;c[d]=h.extend({},m.models.oSearch,c[d]);la(a,d,h(b).data())}function la(a,b,c){var b=a.aoColumns[b],d=a.oClasses,e=h(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=
-e.attr("width")||null;var f=(e.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);f&&(b.sWidthOrig=f[1])}c!==k&&null!==c&&(hb(c),J(m.defaults.column,c),c.mDataProp!==k&&!c.mData&&(c.mData=c.mDataProp),c.sType&&(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),h.extend(b,c),F(b,c,"sWidth","sWidthOrig"),c.iDataSort!==k&&(b.aDataSort=[c.iDataSort]),F(b,c,"aDataSort"));var g=b.mData,j=R(g),i=b.mRender?R(b.mRender):null,c=function(a){return"string"===typeof a&&-1!==a.indexOf("@")};
-b._bAttrSrc=h.isPlainObject(g)&&(c(g.sort)||c(g.type)||c(g.filter));b._setter=null;b.fnGetData=function(a,b,c){var d=j(a,b,k,c);return i&&b?i(d,b,a,c):d};b.fnSetData=function(a,b,c){return S(g)(a,b,c)};"number"!==typeof g&&(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,e.addClass(d.sSortableNone));a=-1!==h.inArray("asc",b.asSorting);c=-1!==h.inArray("desc",b.asSorting);!b.bSortable||!a&&!c?(b.sSortingClass=d.sSortableNone,b.sSortingClassJUI=""):a&&!c?(b.sSortingClass=d.sSortableAsc,b.sSortingClassJUI=
-d.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=d.sSortableDesc,b.sSortingClassJUI=d.sSortJUIDescAllowed):(b.sSortingClass=d.sSortable,b.sSortingClassJUI=d.sSortJUI)}function Z(a){if(!1!==a.oFeatures.bAutoWidth){var b=a.aoColumns;Ha(a);for(var c=0,d=b.length;c<d;c++)b[c].nTh.style.width=b[c].sWidth}b=a.oScroll;(""!==b.sY||""!==b.sX)&&ma(a);s(a,null,"column-sizing",[a])}function $(a,b){var c=na(a,"bVisible");return"number"===typeof c[b]?c[b]:null}function aa(a,b){var c=na(a,"bVisible"),c=h.inArray(b,
-c);return-1!==c?c:null}function ba(a){var b=0;h.each(a.aoColumns,function(a,d){d.bVisible&&"none"!==h(d.nTh).css("display")&&b++});return b}function na(a,b){var c=[];h.map(a.aoColumns,function(a,e){a[b]&&c.push(e)});return c}function Ia(a){var b=a.aoColumns,c=a.aoData,d=m.ext.type.detect,e,f,g,j,i,h,l,q,r;e=0;for(f=b.length;e<f;e++)if(l=b[e],r=[],!l.sType&&l._sManualType)l.sType=l._sManualType;else if(!l.sType){g=0;for(j=d.length;g<j;g++){i=0;for(h=c.length;i<h;i++){r[i]===k&&(r[i]=B(a,i,e,"type"));
-q=d[g](r[i],a);if(!q&&g!==d.length-1)break;if("html"===q)break}if(q){l.sType=q;break}}l.sType||(l.sType="string")}}function kb(a,b,c,d){var e,f,g,j,i,n,l=a.aoColumns;if(b)for(e=b.length-1;0<=e;e--){n=b[e];var q=n.targets!==k?n.targets:n.aTargets;h.isArray(q)||(q=[q]);f=0;for(g=q.length;f<g;f++)if("number"===typeof q[f]&&0<=q[f]){for(;l.length<=q[f];)Ga(a);d(q[f],n)}else if("number"===typeof q[f]&&0>q[f])d(l.length+q[f],n);else if("string"===typeof q[f]){j=0;for(i=l.length;j<i;j++)("_all"==q[f]||h(l[j].nTh).hasClass(q[f]))&&
-d(j,n)}}if(c){e=0;for(a=c.length;e<a;e++)d(e,c[e])}}function N(a,b,c,d){var e=a.aoData.length,f=h.extend(!0,{},m.models.oRow,{src:c?"dom":"data",idx:e});f._aData=b;a.aoData.push(f);for(var g=a.aoColumns,j=0,i=g.length;j<i;j++)g[j].sType=null;a.aiDisplayMaster.push(e);b=a.rowIdFn(b);b!==k&&(a.aIds[b]=f);(c||!a.oFeatures.bDeferRender)&&Ja(a,e,c,d);return e}function oa(a,b){var c;b instanceof h||(b=h(b));return b.map(function(b,e){c=Ka(a,e);return N(a,c.data,e,c.cells)})}function B(a,b,c,d){var e=a.iDraw,
-f=a.aoColumns[c],g=a.aoData[b]._aData,j=f.sDefaultContent,i=f.fnGetData(g,d,{settings:a,row:b,col:c});if(i===k)return a.iDrawError!=e&&null===j&&(K(a,0,"Requested unknown parameter "+("function"==typeof f.mData?"{function}":"'"+f.mData+"'")+" for row "+b+", column "+c,4),a.iDrawError=e),j;if((i===g||null===i)&&null!==j&&d!==k)i=j;else if("function"===typeof i)return i.call(g);return null===i&&"display"==d?"":i}function lb(a,b,c,d){a.aoColumns[c].fnSetData(a.aoData[b]._aData,d,{settings:a,row:b,col:c})}
-function La(a){return h.map(a.match(/(\\.|[^\.])+/g)||[""],function(a){return a.replace(/\\\./g,".")})}function R(a){if(h.isPlainObject(a)){var b={};h.each(a,function(a,c){c&&(b[a]=R(c))});return function(a,c,f,g){var j=b[c]||b._;return j!==k?j(a,c,f,g):a}}if(null===a)return function(a){return a};if("function"===typeof a)return function(b,c,f,g){return a(b,c,f,g)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var c=function(a,b,f){var g,j;if(""!==f){j=La(f);
-for(var i=0,n=j.length;i<n;i++){f=j[i].match(ca);g=j[i].match(V);if(f){j[i]=j[i].replace(ca,"");""!==j[i]&&(a=a[j[i]]);g=[];j.splice(0,i+1);j=j.join(".");if(h.isArray(a)){i=0;for(n=a.length;i<n;i++)g.push(c(a[i],b,j))}a=f[0].substring(1,f[0].length-1);a=""===a?g:g.join(a);break}else if(g){j[i]=j[i].replace(V,"");a=a[j[i]]();continue}if(null===a||a[j[i]]===k)return k;a=a[j[i]]}}return a};return function(b,e){return c(b,e,a)}}return function(b){return b[a]}}function S(a){if(h.isPlainObject(a))return S(a._);
-if(null===a)return function(){};if("function"===typeof a)return function(b,d,e){a(b,"set",d,e)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var b=function(a,d,e){var e=La(e),f;f=e[e.length-1];for(var g,j,i=0,n=e.length-1;i<n;i++){g=e[i].match(ca);j=e[i].match(V);if(g){e[i]=e[i].replace(ca,"");a[e[i]]=[];f=e.slice();f.splice(0,i+1);g=f.join(".");if(h.isArray(d)){j=0;for(n=d.length;j<n;j++)f={},b(f,d[j],g),a[e[i]].push(f)}else a[e[i]]=d;return}j&&(e[i]=e[i].replace(V,
-""),a=a[e[i]](d));if(null===a[e[i]]||a[e[i]]===k)a[e[i]]={};a=a[e[i]]}if(f.match(V))a[f.replace(V,"")](d);else a[f.replace(ca,"")]=d};return function(c,d){return b(c,d,a)}}return function(b,d){b[a]=d}}function Ma(a){return D(a.aoData,"_aData")}function pa(a){a.aoData.length=0;a.aiDisplayMaster.length=0;a.aiDisplay.length=0;a.aIds={}}function qa(a,b,c){for(var d=-1,e=0,f=a.length;e<f;e++)a[e]==b?d=e:a[e]>b&&a[e]--; -1!=d&&c===k&&a.splice(d,1)}function da(a,b,c,d){var e=a.aoData[b],f,g=function(c,d){for(;c.childNodes.length;)c.removeChild(c.firstChild);
-c.innerHTML=B(a,b,d,"display")};if("dom"===c||(!c||"auto"===c)&&"dom"===e.src)e._aData=Ka(a,e,d,d===k?k:e._aData).data;else{var j=e.anCells;if(j)if(d!==k)g(j[d],d);else{c=0;for(f=j.length;c<f;c++)g(j[c],c)}}e._aSortData=null;e._aFilterData=null;g=a.aoColumns;if(d!==k)g[d].sType=null;else{c=0;for(f=g.length;c<f;c++)g[c].sType=null;Na(a,e)}}function Ka(a,b,c,d){var e=[],f=b.firstChild,g,j,i=0,n,l=a.aoColumns,q=a._rowReadObject,d=d!==k?d:q?{}:[],r=function(a,b){if("string"===typeof a){var c=a.indexOf("@");
--1!==c&&(c=a.substring(c+1),S(a)(d,b.getAttribute(c)))}},m=function(a){if(c===k||c===i)j=l[i],n=h.trim(a.innerHTML),j&&j._bAttrSrc?(S(j.mData._)(d,n),r(j.mData.sort,a),r(j.mData.type,a),r(j.mData.filter,a)):q?(j._setter||(j._setter=S(j.mData)),j._setter(d,n)):d[i]=n;i++};if(f)for(;f;){g=f.nodeName.toUpperCase();if("TD"==g||"TH"==g)m(f),e.push(f);f=f.nextSibling}else{e=b.anCells;f=0;for(g=e.length;f<g;f++)m(e[f])}if(b=b.firstChild?b:b.nTr)(b=b.getAttribute("id"))&&S(a.rowId)(d,b);return{data:d,cells:e}}
-function Ja(a,b,c,d){var e=a.aoData[b],f=e._aData,g=[],j,i,n,l,q;if(null===e.nTr){j=c||H.createElement("tr");e.nTr=j;e.anCells=g;j._DT_RowIndex=b;Na(a,e);l=0;for(q=a.aoColumns.length;l<q;l++){n=a.aoColumns[l];i=c?d[l]:H.createElement(n.sCellType);i._DT_CellIndex={row:b,column:l};g.push(i);if((!c||n.mRender||n.mData!==l)&&(!h.isPlainObject(n.mData)||n.mData._!==l+".display"))i.innerHTML=B(a,b,l,"display");n.sClass&&(i.className+=" "+n.sClass);n.bVisible&&!c?j.appendChild(i):!n.bVisible&&c&&i.parentNode.removeChild(i);
-n.fnCreatedCell&&n.fnCreatedCell.call(a.oInstance,i,B(a,b,l),f,b,l)}s(a,"aoRowCreatedCallback",null,[j,f,b])}e.nTr.setAttribute("role","row")}function Na(a,b){var c=b.nTr,d=b._aData;if(c){var e=a.rowIdFn(d);e&&(c.id=e);d.DT_RowClass&&(e=d.DT_RowClass.split(" "),b.__rowc=b.__rowc?sa(b.__rowc.concat(e)):e,h(c).removeClass(b.__rowc.join(" ")).addClass(d.DT_RowClass));d.DT_RowAttr&&h(c).attr(d.DT_RowAttr);d.DT_RowData&&h(c).data(d.DT_RowData)}}function mb(a){var b,c,d,e,f,g=a.nTHead,j=a.nTFoot,i=0===
-h("th, td",g).length,n=a.oClasses,l=a.aoColumns;i&&(e=h("<tr/>").appendTo(g));b=0;for(c=l.length;b<c;b++)f=l[b],d=h(f.nTh).addClass(f.sClass),i&&d.appendTo(e),a.oFeatures.bSort&&(d.addClass(f.sSortingClass),!1!==f.bSortable&&(d.attr("tabindex",a.iTabIndex).attr("aria-controls",a.sTableId),Oa(a,f.nTh,b))),f.sTitle!=d[0].innerHTML&&d.html(f.sTitle),Pa(a,"header")(a,d,f,n);i&&ea(a.aoHeader,g);h(g).find(">tr").attr("role","row");h(g).find(">tr>th, >tr>td").addClass(n.sHeaderTH);h(j).find(">tr>th, >tr>td").addClass(n.sFooterTH);
-if(null!==j){a=a.aoFooter[0];b=0;for(c=a.length;b<c;b++)f=l[b],f.nTf=a[b].cell,f.sClass&&h(f.nTf).addClass(f.sClass)}}function fa(a,b,c){var d,e,f,g=[],j=[],i=a.aoColumns.length,n;if(b){c===k&&(c=!1);d=0;for(e=b.length;d<e;d++){g[d]=b[d].slice();g[d].nTr=b[d].nTr;for(f=i-1;0<=f;f--)!a.aoColumns[f].bVisible&&!c&&g[d].splice(f,1);j.push([])}d=0;for(e=g.length;d<e;d++){if(a=g[d].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[d].length;f<b;f++)if(n=i=1,j[d][f]===k){a.appendChild(g[d][f].cell);
-for(j[d][f]=1;g[d+i]!==k&&g[d][f].cell==g[d+i][f].cell;)j[d+i][f]=1,i++;for(;g[d][f+n]!==k&&g[d][f].cell==g[d][f+n].cell;){for(c=0;c<i;c++)j[d+c][f+n]=1;n++}h(g[d][f].cell).attr("rowspan",i).attr("colspan",n)}}}}function O(a){var b=s(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==h.inArray(!1,b))C(a,!1);else{var b=[],c=0,d=a.asStripeClasses,e=d.length,f=a.oLanguage,g=a.iInitDisplayStart,j="ssp"==y(a),i=a.aiDisplay;a.bDrawing=!0;g!==k&&-1!==g&&(a._iDisplayStart=j?g:g>=a.fnRecordsDisplay()?0:g,a.iInitDisplayStart=
--1);var g=a._iDisplayStart,n=a.fnDisplayEnd();if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,C(a,!1);else if(j){if(!a.bDestroying&&!nb(a))return}else a.iDraw++;if(0!==i.length){f=j?a.aoData.length:n;for(j=j?0:g;j<f;j++){var l=i[j],q=a.aoData[l];null===q.nTr&&Ja(a,l);l=q.nTr;if(0!==e){var r=d[c%e];q._sRowStripe!=r&&(h(l).removeClass(q._sRowStripe).addClass(r),q._sRowStripe=r)}s(a,"aoRowCallback",null,[l,q._aData,c,j]);b.push(l);c++}}else c=f.sZeroRecords,1==a.iDraw&&"ajax"==y(a)?c=f.sLoadingRecords:
-f.sEmptyTable&&0===a.fnRecordsTotal()&&(c=f.sEmptyTable),b[0]=h("<tr/>",{"class":e?d[0]:""}).append(h("<td />",{valign:"top",colSpan:ba(a),"class":a.oClasses.sRowEmpty}).html(c))[0];s(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],Ma(a),g,n,i]);s(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],Ma(a),g,n,i]);d=h(a.nTBody);d.children().detach();d.append(h(b));s(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1}}function T(a,b){var c=a.oFeatures,d=c.bFilter;
-c.bSort&&ob(a);d?ga(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;O(a);a._drawHold=!1}function pb(a){var b=a.oClasses,c=h(a.nTable),c=h("<div/>").insertBefore(c),d=a.oFeatures,e=h("<div/>",{id:a.sTableId+"_wrapper","class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=e[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var f=a.sDom.split(""),g,j,i,n,l,q,k=0;k<f.length;k++){g=null;j=f[k];if("<"==j){i=h("<div/>")[0];
-n=f[k+1];if("'"==n||'"'==n){l="";for(q=2;f[k+q]!=n;)l+=f[k+q],q++;"H"==l?l=b.sJUIHeader:"F"==l&&(l=b.sJUIFooter);-1!=l.indexOf(".")?(n=l.split("."),i.id=n[0].substr(1,n[0].length-1),i.className=n[1]):"#"==l.charAt(0)?i.id=l.substr(1,l.length-1):i.className=l;k+=q}e.append(i);e=h(i)}else if(">"==j)e=e.parent();else if("l"==j&&d.bPaginate&&d.bLengthChange)g=qb(a);else if("f"==j&&d.bFilter)g=rb(a);else if("r"==j&&d.bProcessing)g=sb(a);else if("t"==j)g=tb(a);else if("i"==j&&d.bInfo)g=ub(a);else if("p"==
-j&&d.bPaginate)g=vb(a);else if(0!==m.ext.feature.length){i=m.ext.feature;q=0;for(n=i.length;q<n;q++)if(j==i[q].cFeature){g=i[q].fnInit(a);break}}g&&(i=a.aanFeatures,i[j]||(i[j]=[]),i[j].push(g),e.append(g))}c.replaceWith(e);a.nHolding=null}function ea(a,b){var c=h(b).children("tr"),d,e,f,g,j,i,n,l,q,k;a.splice(0,a.length);f=0;for(i=c.length;f<i;f++)a.push([]);f=0;for(i=c.length;f<i;f++){d=c[f];for(e=d.firstChild;e;){if("TD"==e.nodeName.toUpperCase()||"TH"==e.nodeName.toUpperCase()){l=1*e.getAttribute("colspan");
-q=1*e.getAttribute("rowspan");l=!l||0===l||1===l?1:l;q=!q||0===q||1===q?1:q;g=0;for(j=a[f];j[g];)g++;n=g;k=1===l?!0:!1;for(j=0;j<l;j++)for(g=0;g<q;g++)a[f+g][n+j]={cell:e,unique:k},a[f+g].nTr=d}e=e.nextSibling}}}function ta(a,b,c){var d=[];c||(c=a.aoHeader,b&&(c=[],ea(c,b)));for(var b=0,e=c.length;b<e;b++)for(var f=0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!d[f]||!a.bSortCellsTop))d[f]=c[b][f].cell;return d}function ua(a,b,c){s(a,"aoServerParams","serverParams",[b]);if(b&&h.isArray(b)){var d={},
-e=/(.*?)\[\]$/;h.each(b,function(a,b){var c=b.name.match(e);c?(c=c[0],d[c]||(d[c]=[]),d[c].push(b.value)):d[b.name]=b.value});b=d}var f,g=a.ajax,j=a.oInstance,i=function(b){s(a,null,"xhr",[a,b,a.jqXHR]);c(b)};if(h.isPlainObject(g)&&g.data){f=g.data;var n=h.isFunction(f)?f(b,a):f,b=h.isFunction(f)&&n?n:h.extend(!0,b,n);delete g.data}n={data:b,success:function(b){var c=b.error||b.sError;c&&K(a,0,c);a.json=b;i(b)},dataType:"json",cache:!1,type:a.sServerMethod,error:function(b,c){var d=s(a,null,"xhr",
-[a,null,a.jqXHR]);-1===h.inArray(!0,d)&&("parsererror"==c?K(a,0,"Invalid JSON response",1):4===b.readyState&&K(a,0,"Ajax error",7));C(a,!1)}};a.oAjaxData=b;s(a,null,"preXhr",[a,b]);a.fnServerData?a.fnServerData.call(j,a.sAjaxSource,h.map(b,function(a,b){return{name:b,value:a}}),i,a):a.sAjaxSource||"string"===typeof g?a.jqXHR=h.ajax(h.extend(n,{url:g||a.sAjaxSource})):h.isFunction(g)?a.jqXHR=g.call(j,b,i,a):(a.jqXHR=h.ajax(h.extend(n,g)),g.data=f)}function nb(a){return a.bAjaxDataGet?(a.iDraw++,C(a,
-!0),ua(a,wb(a),function(b){xb(a,b)}),!1):!0}function wb(a){var b=a.aoColumns,c=b.length,d=a.oFeatures,e=a.oPreviousSearch,f=a.aoPreSearchCols,g,j=[],i,n,l,k=W(a);g=a._iDisplayStart;i=!1!==d.bPaginate?a._iDisplayLength:-1;var r=function(a,b){j.push({name:a,value:b})};r("sEcho",a.iDraw);r("iColumns",c);r("sColumns",D(b,"sName").join(","));r("iDisplayStart",g);r("iDisplayLength",i);var ra={draw:a.iDraw,columns:[],order:[],start:g,length:i,search:{value:e.sSearch,regex:e.bRegex}};for(g=0;g<c;g++)n=b[g],
-l=f[g],i="function"==typeof n.mData?"function":n.mData,ra.columns.push({data:i,name:n.sName,searchable:n.bSearchable,orderable:n.bSortable,search:{value:l.sSearch,regex:l.bRegex}}),r("mDataProp_"+g,i),d.bFilter&&(r("sSearch_"+g,l.sSearch),r("bRegex_"+g,l.bRegex),r("bSearchable_"+g,n.bSearchable)),d.bSort&&r("bSortable_"+g,n.bSortable);d.bFilter&&(r("sSearch",e.sSearch),r("bRegex",e.bRegex));d.bSort&&(h.each(k,function(a,b){ra.order.push({column:b.col,dir:b.dir});r("iSortCol_"+a,b.col);r("sSortDir_"+
-a,b.dir)}),r("iSortingCols",k.length));b=m.ext.legacy.ajax;return null===b?a.sAjaxSource?j:ra:b?j:ra}function xb(a,b){var c=va(a,b),d=b.sEcho!==k?b.sEcho:b.draw,e=b.iTotalRecords!==k?b.iTotalRecords:b.recordsTotal,f=b.iTotalDisplayRecords!==k?b.iTotalDisplayRecords:b.recordsFiltered;if(d){if(1*d<a.iDraw)return;a.iDraw=1*d}pa(a);a._iRecordsTotal=parseInt(e,10);a._iRecordsDisplay=parseInt(f,10);d=0;for(e=c.length;d<e;d++)N(a,c[d]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=!1;O(a);a._bInitComplete||
-wa(a,b);a.bAjaxDataGet=!0;C(a,!1)}function va(a,b){var c=h.isPlainObject(a.ajax)&&a.ajax.dataSrc!==k?a.ajax.dataSrc:a.sAjaxDataProp;return"data"===c?b.aaData||b[c]:""!==c?R(c)(b):b}function rb(a){var b=a.oClasses,c=a.sTableId,d=a.oLanguage,e=a.oPreviousSearch,f=a.aanFeatures,g='<input type="search" class="'+b.sFilterInput+'"/>',j=d.sSearch,j=j.match(/_INPUT_/)?j.replace("_INPUT_",g):j+g,b=h("<div/>",{id:!f.f?c+"_filter":null,"class":b.sFilter}).append(h("<label/>").append(j)),f=function(){var b=!this.value?
-"":this.value;b!=e.sSearch&&(ga(a,{sSearch:b,bRegex:e.bRegex,bSmart:e.bSmart,bCaseInsensitive:e.bCaseInsensitive}),a._iDisplayStart=0,O(a))},g=null!==a.searchDelay?a.searchDelay:"ssp"===y(a)?400:0,i=h("input",b).val(e.sSearch).attr("placeholder",d.sSearchPlaceholder).on("keyup.DT search.DT input.DT paste.DT cut.DT",g?Qa(f,g):f).on("keypress.DT",function(a){if(13==a.keyCode)return!1}).attr("aria-controls",c);h(a.nTable).on("search.dt.DT",function(b,c){if(a===c)try{i[0]!==H.activeElement&&i.val(e.sSearch)}catch(d){}});
-return b[0]}function ga(a,b,c){var d=a.oPreviousSearch,e=a.aoPreSearchCols,f=function(a){d.sSearch=a.sSearch;d.bRegex=a.bRegex;d.bSmart=a.bSmart;d.bCaseInsensitive=a.bCaseInsensitive};Ia(a);if("ssp"!=y(a)){yb(a,b.sSearch,c,b.bEscapeRegex!==k?!b.bEscapeRegex:b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<e.length;b++)zb(a,e[b].sSearch,b,e[b].bEscapeRegex!==k?!e[b].bEscapeRegex:e[b].bRegex,e[b].bSmart,e[b].bCaseInsensitive);Ab(a)}else f(b);a.bFiltered=!0;s(a,null,"search",[a])}function Ab(a){for(var b=
-m.ext.search,c=a.aiDisplay,d,e,f=0,g=b.length;f<g;f++){for(var j=[],i=0,n=c.length;i<n;i++)e=c[i],d=a.aoData[e],b[f](a,d._aFilterData,e,d._aData,i)&&j.push(e);c.length=0;h.merge(c,j)}}function zb(a,b,c,d,e,f){if(""!==b){for(var g=[],j=a.aiDisplay,d=Ra(b,d,e,f),e=0;e<j.length;e++)b=a.aoData[j[e]]._aFilterData[c],d.test(b)&&g.push(j[e]);a.aiDisplay=g}}function yb(a,b,c,d,e,f){var d=Ra(b,d,e,f),f=a.oPreviousSearch.sSearch,g=a.aiDisplayMaster,j,e=[];0!==m.ext.search.length&&(c=!0);j=Bb(a);if(0>=b.length)a.aiDisplay=
-g.slice();else{if(j||c||f.length>b.length||0!==b.indexOf(f)||a.bSorted)a.aiDisplay=g.slice();b=a.aiDisplay;for(c=0;c<b.length;c++)d.test(a.aoData[b[c]]._sFilterRow)&&e.push(b[c]);a.aiDisplay=e}}function Ra(a,b,c,d){a=b?a:Sa(a);c&&(a="^(?=.*?"+h.map(a.match(/"[^"]+"|[^ ]+/g)||[""],function(a){if('"'===a.charAt(0))var b=a.match(/^"(.*)"$/),a=b?b[1]:a;return a.replace('"',"")}).join(")(?=.*?")+").*$");return RegExp(a,d?"i":"")}function Bb(a){var b=a.aoColumns,c,d,e,f,g,j,i,h,l=m.ext.type.search;c=!1;
-d=0;for(f=a.aoData.length;d<f;d++)if(h=a.aoData[d],!h._aFilterData){j=[];e=0;for(g=b.length;e<g;e++)c=b[e],c.bSearchable?(i=B(a,d,e,"filter"),l[c.sType]&&(i=l[c.sType](i)),null===i&&(i=""),"string"!==typeof i&&i.toString&&(i=i.toString())):i="",i.indexOf&&-1!==i.indexOf("&")&&(xa.innerHTML=i,i=$b?xa.textContent:xa.innerText),i.replace&&(i=i.replace(/[\r\n]/g,"")),j.push(i);h._aFilterData=j;h._sFilterRow=j.join(" ");c=!0}return c}function Cb(a){return{search:a.sSearch,smart:a.bSmart,regex:a.bRegex,
-caseInsensitive:a.bCaseInsensitive}}function Db(a){return{sSearch:a.search,bSmart:a.smart,bRegex:a.regex,bCaseInsensitive:a.caseInsensitive}}function ub(a){var b=a.sTableId,c=a.aanFeatures.i,d=h("<div/>",{"class":a.oClasses.sInfo,id:!c?b+"_info":null});c||(a.aoDrawCallback.push({fn:Eb,sName:"information"}),d.attr("role","status").attr("aria-live","polite"),h(a.nTable).attr("aria-describedby",b+"_info"));return d[0]}function Eb(a){var b=a.aanFeatures.i;if(0!==b.length){var c=a.oLanguage,d=a._iDisplayStart+
-1,e=a.fnDisplayEnd(),f=a.fnRecordsTotal(),g=a.fnRecordsDisplay(),j=g?c.sInfo:c.sInfoEmpty;g!==f&&(j+=" "+c.sInfoFiltered);j+=c.sInfoPostFix;j=Fb(a,j);c=c.fnInfoCallback;null!==c&&(j=c.call(a.oInstance,a,d,e,f,g,j));h(b).html(j)}}function Fb(a,b){var c=a.fnFormatNumber,d=a._iDisplayStart+1,e=a._iDisplayLength,f=a.fnRecordsDisplay(),g=-1===e;return b.replace(/_START_/g,c.call(a,d)).replace(/_END_/g,c.call(a,a.fnDisplayEnd())).replace(/_MAX_/g,c.call(a,a.fnRecordsTotal())).replace(/_TOTAL_/g,c.call(a,
-f)).replace(/_PAGE_/g,c.call(a,g?1:Math.ceil(d/e))).replace(/_PAGES_/g,c.call(a,g?1:Math.ceil(f/e)))}function ha(a){var b,c,d=a.iInitDisplayStart,e=a.aoColumns,f;c=a.oFeatures;var g=a.bDeferLoading;if(a.bInitialised){pb(a);mb(a);fa(a,a.aoHeader);fa(a,a.aoFooter);C(a,!0);c.bAutoWidth&&Ha(a);b=0;for(c=e.length;b<c;b++)f=e[b],f.sWidth&&(f.nTh.style.width=v(f.sWidth));s(a,null,"preInit",[a]);T(a);e=y(a);if("ssp"!=e||g)"ajax"==e?ua(a,[],function(c){var f=va(a,c);for(b=0;b<f.length;b++)N(a,f[b]);a.iInitDisplayStart=
-d;T(a);C(a,!1);wa(a,c)},a):(C(a,!1),wa(a))}else setTimeout(function(){ha(a)},200)}function wa(a,b){a._bInitComplete=!0;(b||a.oInit.aaData)&&Z(a);s(a,null,"plugin-init",[a,b]);s(a,"aoInitComplete","init",[a,b])}function Ta(a,b){var c=parseInt(b,10);a._iDisplayLength=c;Ua(a);s(a,null,"length",[a,c])}function qb(a){for(var b=a.oClasses,c=a.sTableId,d=a.aLengthMenu,e=h.isArray(d[0]),f=e?d[0]:d,d=e?d[1]:d,e=h("<select/>",{name:c+"_length","aria-controls":c,"class":b.sLengthSelect}),g=0,j=f.length;g<j;g++)e[0][g]=
-new Option(d[g],f[g]);var i=h("<div><label/></div>").addClass(b.sLength);a.aanFeatures.l||(i[0].id=c+"_length");i.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",e[0].outerHTML));h("select",i).val(a._iDisplayLength).on("change.DT",function(){Ta(a,h(this).val());O(a)});h(a.nTable).on("length.dt.DT",function(b,c,d){a===c&&h("select",i).val(d)});return i[0]}function vb(a){var b=a.sPaginationType,c=m.ext.pager[b],d="function"===typeof c,e=function(a){O(a)},b=h("<div/>").addClass(a.oClasses.sPaging+
-b)[0],f=a.aanFeatures;d||c.fnInit(a,b,e);f.p||(b.id=a.sTableId+"_paginate",a.aoDrawCallback.push({fn:function(a){if(d){var b=a._iDisplayStart,i=a._iDisplayLength,h=a.fnRecordsDisplay(),l=-1===i,b=l?0:Math.ceil(b/i),i=l?1:Math.ceil(h/i),h=c(b,i),k,l=0;for(k=f.p.length;l<k;l++)Pa(a,"pageButton")(a,f.p[l],l,h,b,i)}else c.fnUpdate(a,e)},sName:"pagination"}));return b}function Va(a,b,c){var d=a._iDisplayStart,e=a._iDisplayLength,f=a.fnRecordsDisplay();0===f||-1===e?d=0:"number"===typeof b?(d=b*e,d>f&&
-(d=0)):"first"==b?d=0:"previous"==b?(d=0<=e?d-e:0,0>d&&(d=0)):"next"==b?d+e<f&&(d+=e):"last"==b?d=Math.floor((f-1)/e)*e:K(a,0,"Unknown paging action: "+b,5);b=a._iDisplayStart!==d;a._iDisplayStart=d;b&&(s(a,null,"page",[a]),c&&O(a));return b}function sb(a){return h("<div/>",{id:!a.aanFeatures.r?a.sTableId+"_processing":null,"class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]}function C(a,b){a.oFeatures.bProcessing&&h(a.aanFeatures.r).css("display",b?"block":"none");
-s(a,null,"processing",[a,b])}function tb(a){var b=h(a.nTable);b.attr("role","grid");var c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var d=c.sX,e=c.sY,f=a.oClasses,g=b.children("caption"),j=g.length?g[0]._captionSide:null,i=h(b[0].cloneNode(!1)),n=h(b[0].cloneNode(!1)),l=b.children("tfoot");l.length||(l=null);i=h("<div/>",{"class":f.sScrollWrapper}).append(h("<div/>",{"class":f.sScrollHead}).css({overflow:"hidden",position:"relative",border:0,width:d?!d?null:v(d):"100%"}).append(h("<div/>",
-{"class":f.sScrollHeadInner}).css({"box-sizing":"content-box",width:c.sXInner||"100%"}).append(i.removeAttr("id").css("margin-left",0).append("top"===j?g:null).append(b.children("thead"))))).append(h("<div/>",{"class":f.sScrollBody}).css({position:"relative",overflow:"auto",width:!d?null:v(d)}).append(b));l&&i.append(h("<div/>",{"class":f.sScrollFoot}).css({overflow:"hidden",border:0,width:d?!d?null:v(d):"100%"}).append(h("<div/>",{"class":f.sScrollFootInner}).append(n.removeAttr("id").css("margin-left",
-0).append("bottom"===j?g:null).append(b.children("tfoot")))));var b=i.children(),k=b[0],f=b[1],r=l?b[2]:null;if(d)h(f).on("scroll.DT",function(){var a=this.scrollLeft;k.scrollLeft=a;l&&(r.scrollLeft=a)});h(f).css(e&&c.bCollapse?"max-height":"height",e);a.nScrollHead=k;a.nScrollBody=f;a.nScrollFoot=r;a.aoDrawCallback.push({fn:ma,sName:"scrolling"});return i[0]}function ma(a){var b=a.oScroll,c=b.sX,d=b.sXInner,e=b.sY,b=b.iBarWidth,f=h(a.nScrollHead),g=f[0].style,j=f.children("div"),i=j[0].style,n=j.children("table"),
-j=a.nScrollBody,l=h(j),q=j.style,r=h(a.nScrollFoot).children("div"),m=r.children("table"),p=h(a.nTHead),o=h(a.nTable),t=o[0],s=t.style,u=a.nTFoot?h(a.nTFoot):null,x=a.oBrowser,U=x.bScrollOversize,ac=D(a.aoColumns,"nTh"),P,L,Q,w,Wa=[],y=[],z=[],A=[],B,C=function(a){a=a.style;a.paddingTop="0";a.paddingBottom="0";a.borderTopWidth="0";a.borderBottomWidth="0";a.height=0};L=j.scrollHeight>j.clientHeight;if(a.scrollBarVis!==L&&a.scrollBarVis!==k)a.scrollBarVis=L,Z(a);else{a.scrollBarVis=L;o.children("thead, tfoot").remove();
-u&&(Q=u.clone().prependTo(o),P=u.find("tr"),Q=Q.find("tr"));w=p.clone().prependTo(o);p=p.find("tr");L=w.find("tr");w.find("th, td").removeAttr("tabindex");c||(q.width="100%",f[0].style.width="100%");h.each(ta(a,w),function(b,c){B=$(a,b);c.style.width=a.aoColumns[B].sWidth});u&&I(function(a){a.style.width=""},Q);f=o.outerWidth();if(""===c){s.width="100%";if(U&&(o.find("tbody").height()>j.offsetHeight||"scroll"==l.css("overflow-y")))s.width=v(o.outerWidth()-b);f=o.outerWidth()}else""!==d&&(s.width=
-v(d),f=o.outerWidth());I(C,L);I(function(a){z.push(a.innerHTML);Wa.push(v(h(a).css("width")))},L);I(function(a,b){if(h.inArray(a,ac)!==-1)a.style.width=Wa[b]},p);h(L).height(0);u&&(I(C,Q),I(function(a){A.push(a.innerHTML);y.push(v(h(a).css("width")))},Q),I(function(a,b){a.style.width=y[b]},P),h(Q).height(0));I(function(a,b){a.innerHTML='<div class="dataTables_sizing" style="height:0;overflow:hidden;">'+z[b]+"</div>";a.style.width=Wa[b]},L);u&&I(function(a,b){a.innerHTML='<div class="dataTables_sizing" style="height:0;overflow:hidden;">'+
-A[b]+"</div>";a.style.width=y[b]},Q);if(o.outerWidth()<f){P=j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")?f+b:f;if(U&&(j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")))s.width=v(P-b);(""===c||""!==d)&&K(a,1,"Possible column misalignment",6)}else P="100%";q.width=v(P);g.width=v(P);u&&(a.nScrollFoot.style.width=v(P));!e&&U&&(q.height=v(t.offsetHeight+b));c=o.outerWidth();n[0].style.width=v(c);i.width=v(c);d=o.height()>j.clientHeight||"scroll"==l.css("overflow-y");e="padding"+
-(x.bScrollbarLeft?"Left":"Right");i[e]=d?b+"px":"0px";u&&(m[0].style.width=v(c),r[0].style.width=v(c),r[0].style[e]=d?b+"px":"0px");o.children("colgroup").insertBefore(o.children("thead"));l.scroll();if((a.bSorted||a.bFiltered)&&!a._drawHold)j.scrollTop=0}}function I(a,b,c){for(var d=0,e=0,f=b.length,g,j;e<f;){g=b[e].firstChild;for(j=c?c[e].firstChild:null;g;)1===g.nodeType&&(c?a(g,j,d):a(g,d),d++),g=g.nextSibling,j=c?j.nextSibling:null;e++}}function Ha(a){var b=a.nTable,c=a.aoColumns,d=a.oScroll,
-e=d.sY,f=d.sX,g=d.sXInner,j=c.length,i=na(a,"bVisible"),n=h("th",a.nTHead),l=b.getAttribute("width"),k=b.parentNode,r=!1,m,p,o=a.oBrowser,d=o.bScrollOversize;(m=b.style.width)&&-1!==m.indexOf("%")&&(l=m);for(m=0;m<i.length;m++)p=c[i[m]],null!==p.sWidth&&(p.sWidth=Gb(p.sWidthOrig,k),r=!0);if(d||!r&&!f&&!e&&j==ba(a)&&j==n.length)for(m=0;m<j;m++)i=$(a,m),null!==i&&(c[i].sWidth=v(n.eq(m).width()));else{j=h(b).clone().css("visibility","hidden").removeAttr("id");j.find("tbody tr").remove();var t=h("<tr/>").appendTo(j.find("tbody"));
-j.find("thead, tfoot").remove();j.append(h(a.nTHead).clone()).append(h(a.nTFoot).clone());j.find("tfoot th, tfoot td").css("width","");n=ta(a,j.find("thead")[0]);for(m=0;m<i.length;m++)p=c[i[m]],n[m].style.width=null!==p.sWidthOrig&&""!==p.sWidthOrig?v(p.sWidthOrig):"",p.sWidthOrig&&f&&h(n[m]).append(h("<div/>").css({width:p.sWidthOrig,margin:0,padding:0,border:0,height:1}));if(a.aoData.length)for(m=0;m<i.length;m++)r=i[m],p=c[r],h(Hb(a,r)).clone(!1).append(p.sContentPadding).appendTo(t);h("[name]",
-j).removeAttr("name");p=h("<div/>").css(f||e?{position:"absolute",top:0,left:0,height:1,right:0,overflow:"hidden"}:{}).append(j).appendTo(k);f&&g?j.width(g):f?(j.css("width","auto"),j.removeAttr("width"),j.width()<k.clientWidth&&l&&j.width(k.clientWidth)):e?j.width(k.clientWidth):l&&j.width(l);for(m=e=0;m<i.length;m++)k=h(n[m]),g=k.outerWidth()-k.width(),k=o.bBounding?Math.ceil(n[m].getBoundingClientRect().width):k.outerWidth(),e+=k,c[i[m]].sWidth=v(k-g);b.style.width=v(e);p.remove()}l&&(b.style.width=
-v(l));if((l||f)&&!a._reszEvt)b=function(){h(E).on("resize.DT-"+a.sInstance,Qa(function(){Z(a)}))},d?setTimeout(b,1E3):b(),a._reszEvt=!0}function Gb(a,b){if(!a)return 0;var c=h("<div/>").css("width",v(a)).appendTo(b||H.body),d=c[0].offsetWidth;c.remove();return d}function Hb(a,b){var c=Ib(a,b);if(0>c)return null;var d=a.aoData[c];return!d.nTr?h("<td/>").html(B(a,c,b,"display"))[0]:d.anCells[b]}function Ib(a,b){for(var c,d=-1,e=-1,f=0,g=a.aoData.length;f<g;f++)c=B(a,f,b,"display")+"",c=c.replace(bc,
-""),c=c.replace(/ /g," "),c.length>d&&(d=c.length,e=f);return e}function v(a){return null===a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function W(a){var b,c,d=[],e=a.aoColumns,f,g,j,i;b=a.aaSortingFixed;c=h.isPlainObject(b);var n=[];f=function(a){a.length&&!h.isArray(a[0])?n.push(a):h.merge(n,a)};h.isArray(b)&&f(b);c&&b.pre&&f(b.pre);f(a.aaSorting);c&&b.post&&f(b.post);for(a=0;a<n.length;a++){i=n[a][0];f=e[i].aDataSort;b=0;for(c=f.length;b<c;b++)g=f[b],j=e[g].sType||
-"string",n[a]._idx===k&&(n[a]._idx=h.inArray(n[a][1],e[g].asSorting)),d.push({src:i,col:g,dir:n[a][1],index:n[a]._idx,type:j,formatter:m.ext.type.order[j+"-pre"]})}return d}function ob(a){var b,c,d=[],e=m.ext.type.order,f=a.aoData,g=0,j,i=a.aiDisplayMaster,h;Ia(a);h=W(a);b=0;for(c=h.length;b<c;b++)j=h[b],j.formatter&&g++,Jb(a,j.col);if("ssp"!=y(a)&&0!==h.length){b=0;for(c=i.length;b<c;b++)d[i[b]]=b;g===h.length?i.sort(function(a,b){var c,e,g,j,i=h.length,k=f[a]._aSortData,m=f[b]._aSortData;for(g=
-0;g<i;g++)if(j=h[g],c=k[j.col],e=m[j.col],c=c<e?-1:c>e?1:0,0!==c)return"asc"===j.dir?c:-c;c=d[a];e=d[b];return c<e?-1:c>e?1:0}):i.sort(function(a,b){var c,g,j,i,k=h.length,m=f[a]._aSortData,p=f[b]._aSortData;for(j=0;j<k;j++)if(i=h[j],c=m[i.col],g=p[i.col],i=e[i.type+"-"+i.dir]||e["string-"+i.dir],c=i(c,g),0!==c)return c;c=d[a];g=d[b];return c<g?-1:c>g?1:0})}a.bSorted=!0}function Kb(a){for(var b,c,d=a.aoColumns,e=W(a),a=a.oLanguage.oAria,f=0,g=d.length;f<g;f++){c=d[f];var j=c.asSorting;b=c.sTitle.replace(/<.*?>/g,
-"");var i=c.nTh;i.removeAttribute("aria-sort");c.bSortable&&(0<e.length&&e[0].col==f?(i.setAttribute("aria-sort","asc"==e[0].dir?"ascending":"descending"),c=j[e[0].index+1]||j[0]):c=j[0],b+="asc"===c?a.sSortAscending:a.sSortDescending);i.setAttribute("aria-label",b)}}function Xa(a,b,c,d){var e=a.aaSorting,f=a.aoColumns[b].asSorting,g=function(a,b){var c=a._idx;c===k&&(c=h.inArray(a[1],f));return c+1<f.length?c+1:b?null:0};"number"===typeof e[0]&&(e=a.aaSorting=[e]);c&&a.oFeatures.bSortMulti?(c=h.inArray(b,
-D(e,"0")),-1!==c?(b=g(e[c],!0),null===b&&1===e.length&&(b=0),null===b?e.splice(c,1):(e[c][1]=f[b],e[c]._idx=b)):(e.push([b,f[0],0]),e[e.length-1]._idx=0)):e.length&&e[0][0]==b?(b=g(e[0]),e.length=1,e[0][1]=f[b],e[0]._idx=b):(e.length=0,e.push([b,f[0]]),e[0]._idx=0);T(a);"function"==typeof d&&d(a)}function Oa(a,b,c,d){var e=a.aoColumns[c];Ya(b,{},function(b){!1!==e.bSortable&&(a.oFeatures.bProcessing?(C(a,!0),setTimeout(function(){Xa(a,c,b.shiftKey,d);"ssp"!==y(a)&&C(a,!1)},0)):Xa(a,c,b.shiftKey,d))})}
-function ya(a){var b=a.aLastSort,c=a.oClasses.sSortColumn,d=W(a),e=a.oFeatures,f,g;if(e.bSort&&e.bSortClasses){e=0;for(f=b.length;e<f;e++)g=b[e].src,h(D(a.aoData,"anCells",g)).removeClass(c+(2>e?e+1:3));e=0;for(f=d.length;e<f;e++)g=d[e].src,h(D(a.aoData,"anCells",g)).addClass(c+(2>e?e+1:3))}a.aLastSort=d}function Jb(a,b){var c=a.aoColumns[b],d=m.ext.order[c.sSortDataType],e;d&&(e=d.call(a.oInstance,a,b,aa(a,b)));for(var f,g=m.ext.type.order[c.sType+"-pre"],j=0,i=a.aoData.length;j<i;j++)if(c=a.aoData[j],
-c._aSortData||(c._aSortData=[]),!c._aSortData[b]||d)f=d?e[j]:B(a,j,b,"sort"),c._aSortData[b]=g?g(f):f}function za(a){if(a.oFeatures.bStateSave&&!a.bDestroying){var b={time:+new Date,start:a._iDisplayStart,length:a._iDisplayLength,order:h.extend(!0,[],a.aaSorting),search:Cb(a.oPreviousSearch),columns:h.map(a.aoColumns,function(b,d){return{visible:b.bVisible,search:Cb(a.aoPreSearchCols[d])}})};s(a,"aoStateSaveParams","stateSaveParams",[a,b]);a.oSavedState=b;a.fnStateSaveCallback.call(a.oInstance,a,
-b)}}function Lb(a,b,c){var d,e,f=a.aoColumns,b=function(b){if(b&&b.time){var g=s(a,"aoStateLoadParams","stateLoadParams",[a,b]);if(-1===h.inArray(!1,g)&&(g=a.iStateDuration,!(0<g&&b.time<+new Date-1E3*g)&&!(b.columns&&f.length!==b.columns.length))){a.oLoadedState=h.extend(!0,{},b);b.start!==k&&(a._iDisplayStart=b.start,a.iInitDisplayStart=b.start);b.length!==k&&(a._iDisplayLength=b.length);b.order!==k&&(a.aaSorting=[],h.each(b.order,function(b,c){a.aaSorting.push(c[0]>=f.length?[0,c[1]]:c)}));b.search!==
-k&&h.extend(a.oPreviousSearch,Db(b.search));if(b.columns){d=0;for(e=b.columns.length;d<e;d++)g=b.columns[d],g.visible!==k&&(f[d].bVisible=g.visible),g.search!==k&&h.extend(a.aoPreSearchCols[d],Db(g.search))}s(a,"aoStateLoaded","stateLoaded",[a,b])}}c()};if(a.oFeatures.bStateSave){var g=a.fnStateLoadCallback.call(a.oInstance,a,b);g!==k&&b(g)}else c()}function Aa(a){var b=m.settings,a=h.inArray(a,D(b,"nTable"));return-1!==a?b[a]:null}function K(a,b,c,d){c="DataTables warning: "+(a?"table id="+a.sTableId+
-" - ":"")+c;d&&(c+=". For more information about this error, please see http://datatables.net/tn/"+d);if(b)E.console&&console.log&&console.log(c);else if(b=m.ext,b=b.sErrMode||b.errMode,a&&s(a,null,"error",[a,d,c]),"alert"==b)alert(c);else{if("throw"==b)throw Error(c);"function"==typeof b&&b(a,d,c)}}function F(a,b,c,d){h.isArray(c)?h.each(c,function(c,d){h.isArray(d)?F(a,b,d[0],d[1]):F(a,b,d)}):(d===k&&(d=c),b[c]!==k&&(a[d]=b[c]))}function Mb(a,b,c){var d,e;for(e in b)b.hasOwnProperty(e)&&(d=b[e],
-h.isPlainObject(d)?(h.isPlainObject(a[e])||(a[e]={}),h.extend(!0,a[e],d)):a[e]=c&&"data"!==e&&"aaData"!==e&&h.isArray(d)?d.slice():d);return a}function Ya(a,b,c){h(a).on("click.DT",b,function(b){a.blur();c(b)}).on("keypress.DT",b,function(a){13===a.which&&(a.preventDefault(),c(a))}).on("selectstart.DT",function(){return!1})}function z(a,b,c,d){c&&a[b].push({fn:c,sName:d})}function s(a,b,c,d){var e=[];b&&(e=h.map(a[b].slice().reverse(),function(b){return b.fn.apply(a.oInstance,d)}));null!==c&&(b=h.Event(c+
-".dt"),h(a.nTable).trigger(b,d),e.push(b.result));return e}function Ua(a){var b=a._iDisplayStart,c=a.fnDisplayEnd(),d=a._iDisplayLength;b>=c&&(b=c-d);b-=b%d;if(-1===d||0>b)b=0;a._iDisplayStart=b}function Pa(a,b){var c=a.renderer,d=m.ext.renderer[b];return h.isPlainObject(c)&&c[b]?d[c[b]]||d._:"string"===typeof c?d[c]||d._:d._}function y(a){return a.oFeatures.bServerSide?"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function ia(a,b){var c=[],c=Nb.numbers_length,d=Math.floor(c/2);b<=c?c=X(0,b):a<=d?(c=X(0,
-c-2),c.push("ellipsis"),c.push(b-1)):(a>=b-1-d?c=X(b-(c-2),b):(c=X(a-d+2,a+d-1),c.push("ellipsis"),c.push(b-1)),c.splice(0,0,"ellipsis"),c.splice(0,0,0));c.DT_el="span";return c}function fb(a){h.each({num:function(b){return Ba(b,a)},"num-fmt":function(b){return Ba(b,a,Za)},"html-num":function(b){return Ba(b,a,Ca)},"html-num-fmt":function(b){return Ba(b,a,Ca,Za)}},function(b,c){x.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(x.type.search[b+a]=x.type.search.html)})}function Ob(a){return function(){var b=
-[Aa(this[m.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return m.ext.internal[a].apply(this,b)}}var m=function(a){this.$=function(a,b){return this.api(!0).$(a,b)};this._=function(a,b){return this.api(!0).rows(a,b).data()};this.api=function(a){return a?new t(Aa(this[x.iApiIndex])):new t(this)};this.fnAddData=function(a,b){var c=this.api(!0),d=h.isArray(a)&&(h.isArray(a[0])||h.isPlainObject(a[0]))?c.rows.add(a):c.row.add(a);(b===k||b)&&c.draw();return d.flatten().toArray()};this.fnAdjustColumnSizing=
-function(a){var b=this.api(!0).columns.adjust(),c=b.settings()[0],d=c.oScroll;a===k||a?b.draw(!1):(""!==d.sX||""!==d.sY)&&ma(c)};this.fnClearTable=function(a){var b=this.api(!0).clear();(a===k||a)&&b.draw()};this.fnClose=function(a){this.api(!0).row(a).child.hide()};this.fnDeleteRow=function(a,b,c){var d=this.api(!0),a=d.rows(a),e=a.settings()[0],h=e.aoData[a[0][0]];a.remove();b&&b.call(this,e,h);(c===k||c)&&d.draw();return h};this.fnDestroy=function(a){this.api(!0).destroy(a)};this.fnDraw=function(a){this.api(!0).draw(a)};
-this.fnFilter=function(a,b,c,d,e,h){e=this.api(!0);null===b||b===k?e.search(a,c,d,h):e.column(b).search(a,c,d,h);e.draw()};this.fnGetData=function(a,b){var c=this.api(!0);if(a!==k){var d=a.nodeName?a.nodeName.toLowerCase():"";return b!==k||"td"==d||"th"==d?c.cell(a,b).data():c.row(a).data()||null}return c.data().toArray()};this.fnGetNodes=function(a){var b=this.api(!0);return a!==k?b.row(a).node():b.rows().nodes().flatten().toArray()};this.fnGetPosition=function(a){var b=this.api(!0),c=a.nodeName.toUpperCase();
-return"TR"==c?b.row(a).index():"TD"==c||"TH"==c?(a=b.cell(a).index(),[a.row,a.columnVisible,a.column]):null};this.fnIsOpen=function(a){return this.api(!0).row(a).child.isShown()};this.fnOpen=function(a,b,c){return this.api(!0).row(a).child(b,c).show().child()[0]};this.fnPageChange=function(a,b){var c=this.api(!0).page(a);(b===k||b)&&c.draw(!1)};this.fnSetColumnVis=function(a,b,c){a=this.api(!0).column(a).visible(b);(c===k||c)&&a.columns.adjust().draw()};this.fnSettings=function(){return Aa(this[x.iApiIndex])};
-this.fnSort=function(a){this.api(!0).order(a).draw()};this.fnSortListener=function(a,b,c){this.api(!0).order.listener(a,b,c)};this.fnUpdate=function(a,b,c,d,e){var h=this.api(!0);c===k||null===c?h.row(b).data(a):h.cell(b,c).data(a);(e===k||e)&&h.columns.adjust();(d===k||d)&&h.draw();return 0};this.fnVersionCheck=x.fnVersionCheck;var b=this,c=a===k,d=this.length;c&&(a={});this.oApi=this.internal=x.internal;for(var e in m.ext.internal)e&&(this[e]=Ob(e));this.each(function(){var e={},g=1<d?Mb(e,a,!0):
-a,j=0,i,e=this.getAttribute("id"),n=!1,l=m.defaults,q=h(this);if("table"!=this.nodeName.toLowerCase())K(null,0,"Non-table node initialisation ("+this.nodeName+")",2);else{gb(l);hb(l.column);J(l,l,!0);J(l.column,l.column,!0);J(l,h.extend(g,q.data()));var r=m.settings,j=0;for(i=r.length;j<i;j++){var p=r[j];if(p.nTable==this||p.nTHead.parentNode==this||p.nTFoot&&p.nTFoot.parentNode==this){var t=g.bRetrieve!==k?g.bRetrieve:l.bRetrieve;if(c||t)return p.oInstance;if(g.bDestroy!==k?g.bDestroy:l.bDestroy){p.oInstance.fnDestroy();
-break}else{K(p,0,"Cannot reinitialise DataTable",3);return}}if(p.sTableId==this.id){r.splice(j,1);break}}if(null===e||""===e)this.id=e="DataTables_Table_"+m.ext._unique++;var o=h.extend(!0,{},m.models.oSettings,{sDestroyWidth:q[0].style.width,sInstance:e,sTableId:e});o.nTable=this;o.oApi=b.internal;o.oInit=g;r.push(o);o.oInstance=1===b.length?b:q.dataTable();gb(g);g.oLanguage&&Fa(g.oLanguage);g.aLengthMenu&&!g.iDisplayLength&&(g.iDisplayLength=h.isArray(g.aLengthMenu[0])?g.aLengthMenu[0][0]:g.aLengthMenu[0]);
-g=Mb(h.extend(!0,{},l),g);F(o.oFeatures,g,"bPaginate bLengthChange bFilter bSort bSortMulti bInfo bProcessing bAutoWidth bSortClasses bServerSide bDeferRender".split(" "));F(o,g,["asStripeClasses","ajax","fnServerData","fnFormatNumber","sServerMethod","aaSorting","aaSortingFixed","aLengthMenu","sPaginationType","sAjaxSource","sAjaxDataProp","iStateDuration","sDom","bSortCellsTop","iTabIndex","fnStateLoadCallback","fnStateSaveCallback","renderer","searchDelay","rowId",["iCookieDuration","iStateDuration"],
-["oSearch","oPreviousSearch"],["aoSearchCols","aoPreSearchCols"],["iDisplayLength","_iDisplayLength"],["bJQueryUI","bJUI"]]);F(o.oScroll,g,[["sScrollX","sX"],["sScrollXInner","sXInner"],["sScrollY","sY"],["bScrollCollapse","bCollapse"]]);F(o.oLanguage,g,"fnInfoCallback");z(o,"aoDrawCallback",g.fnDrawCallback,"user");z(o,"aoServerParams",g.fnServerParams,"user");z(o,"aoStateSaveParams",g.fnStateSaveParams,"user");z(o,"aoStateLoadParams",g.fnStateLoadParams,"user");z(o,"aoStateLoaded",g.fnStateLoaded,
-"user");z(o,"aoRowCallback",g.fnRowCallback,"user");z(o,"aoRowCreatedCallback",g.fnCreatedRow,"user");z(o,"aoHeaderCallback",g.fnHeaderCallback,"user");z(o,"aoFooterCallback",g.fnFooterCallback,"user");z(o,"aoInitComplete",g.fnInitComplete,"user");z(o,"aoPreDrawCallback",g.fnPreDrawCallback,"user");o.rowIdFn=R(g.rowId);ib(o);var u=o.oClasses;g.bJQueryUI?(h.extend(u,m.ext.oJUIClasses,g.oClasses),g.sDom===l.sDom&&"lfrtip"===l.sDom&&(o.sDom='<"H"lfr>t<"F"ip>'),o.renderer)?h.isPlainObject(o.renderer)&&
-!o.renderer.header&&(o.renderer.header="jqueryui"):o.renderer="jqueryui":h.extend(u,m.ext.classes,g.oClasses);q.addClass(u.sTable);o.iInitDisplayStart===k&&(o.iInitDisplayStart=g.iDisplayStart,o._iDisplayStart=g.iDisplayStart);null!==g.iDeferLoading&&(o.bDeferLoading=!0,e=h.isArray(g.iDeferLoading),o._iRecordsDisplay=e?g.iDeferLoading[0]:g.iDeferLoading,o._iRecordsTotal=e?g.iDeferLoading[1]:g.iDeferLoading);var v=o.oLanguage;h.extend(!0,v,g.oLanguage);v.sUrl&&(h.ajax({dataType:"json",url:v.sUrl,success:function(a){Fa(a);
-J(l.oLanguage,a);h.extend(true,v,a);ha(o)},error:function(){ha(o)}}),n=!0);null===g.asStripeClasses&&(o.asStripeClasses=[u.sStripeOdd,u.sStripeEven]);var e=o.asStripeClasses,x=q.children("tbody").find("tr").eq(0);-1!==h.inArray(!0,h.map(e,function(a){return x.hasClass(a)}))&&(h("tbody tr",this).removeClass(e.join(" ")),o.asDestroyStripes=e.slice());e=[];r=this.getElementsByTagName("thead");0!==r.length&&(ea(o.aoHeader,r[0]),e=ta(o));if(null===g.aoColumns){r=[];j=0;for(i=e.length;j<i;j++)r.push(null)}else r=
-g.aoColumns;j=0;for(i=r.length;j<i;j++)Ga(o,e?e[j]:null);kb(o,g.aoColumnDefs,r,function(a,b){la(o,a,b)});if(x.length){var w=function(a,b){return a.getAttribute("data-"+b)!==null?b:null};h(x[0]).children("th, td").each(function(a,b){var c=o.aoColumns[a];if(c.mData===a){var d=w(b,"sort")||w(b,"order"),e=w(b,"filter")||w(b,"search");if(d!==null||e!==null){c.mData={_:a+".display",sort:d!==null?a+".@data-"+d:k,type:d!==null?a+".@data-"+d:k,filter:e!==null?a+".@data-"+e:k};la(o,a)}}})}var U=o.oFeatures,
-e=function(){if(g.aaSorting===k){var a=o.aaSorting;j=0;for(i=a.length;j<i;j++)a[j][1]=o.aoColumns[j].asSorting[0]}ya(o);U.bSort&&z(o,"aoDrawCallback",function(){if(o.bSorted){var a=W(o),b={};h.each(a,function(a,c){b[c.src]=c.dir});s(o,null,"order",[o,a,b]);Kb(o)}});z(o,"aoDrawCallback",function(){(o.bSorted||y(o)==="ssp"||U.bDeferRender)&&ya(o)},"sc");var a=q.children("caption").each(function(){this._captionSide=h(this).css("caption-side")}),b=q.children("thead");b.length===0&&(b=h("<thead/>").appendTo(q));
-o.nTHead=b[0];b=q.children("tbody");b.length===0&&(b=h("<tbody/>").appendTo(q));o.nTBody=b[0];b=q.children("tfoot");if(b.length===0&&a.length>0&&(o.oScroll.sX!==""||o.oScroll.sY!==""))b=h("<tfoot/>").appendTo(q);if(b.length===0||b.children().length===0)q.addClass(u.sNoFooter);else if(b.length>0){o.nTFoot=b[0];ea(o.aoFooter,o.nTFoot)}if(g.aaData)for(j=0;j<g.aaData.length;j++)N(o,g.aaData[j]);else(o.bDeferLoading||y(o)=="dom")&&oa(o,h(o.nTBody).children("tr"));o.aiDisplay=o.aiDisplayMaster.slice();
-o.bInitialised=true;n===false&&ha(o)};g.bStateSave?(U.bStateSave=!0,z(o,"aoDrawCallback",za,"state_save"),Lb(o,g,e)):e()}});b=null;return this},x,t,p,u,$a={},Pb=/[\r\n]/g,Ca=/<.*?>/g,cc=/^\d{2,4}[\.\/\-]\d{1,2}[\.\/\-]\d{1,2}([T ]{1}\d{1,2}[:\.]\d{2}([\.:]\d{2})?)?$/,dc=RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),Za=/[',$£€¥%\u2009\u202F\u20BD\u20a9\u20BArfk]/gi,M=function(a){return!a||!0===a||"-"===a?!0:!1},Qb=function(a){var b=parseInt(a,10);return!isNaN(b)&&
-isFinite(a)?b:null},Rb=function(a,b){$a[b]||($a[b]=RegExp(Sa(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace($a[b],"."):a},ab=function(a,b,c){var d="string"===typeof a;if(M(a))return!0;b&&d&&(a=Rb(a,b));c&&d&&(a=a.replace(Za,""));return!isNaN(parseFloat(a))&&isFinite(a)},Sb=function(a,b,c){return M(a)?!0:!(M(a)||"string"===typeof a)?null:ab(a.replace(Ca,""),b,c)?!0:null},D=function(a,b,c){var d=[],e=0,f=a.length;if(c!==k)for(;e<f;e++)a[e]&&a[e][b]&&d.push(a[e][b][c]);else for(;e<
-f;e++)a[e]&&d.push(a[e][b]);return d},ja=function(a,b,c,d){var e=[],f=0,g=b.length;if(d!==k)for(;f<g;f++)a[b[f]][c]&&e.push(a[b[f]][c][d]);else for(;f<g;f++)e.push(a[b[f]][c]);return e},X=function(a,b){var c=[],d;b===k?(b=0,d=a):(d=b,b=a);for(var e=b;e<d;e++)c.push(e);return c},Tb=function(a){for(var b=[],c=0,d=a.length;c<d;c++)a[c]&&b.push(a[c]);return b},sa=function(a){var b;a:{if(!(2>a.length)){b=a.slice().sort();for(var c=b[0],d=1,e=b.length;d<e;d++){if(b[d]===c){b=!1;break a}c=b[d]}}b=!0}if(b)return a.slice();
-b=[];var e=a.length,f,g=0,d=0;a:for(;d<e;d++){c=a[d];for(f=0;f<g;f++)if(b[f]===c)continue a;b.push(c);g++}return b};m.util={throttle:function(a,b){var c=b!==k?b:200,d,e;return function(){var b=this,g=+new Date,h=arguments;d&&g<d+c?(clearTimeout(e),e=setTimeout(function(){d=k;a.apply(b,h)},c)):(d=g,a.apply(b,h))}},escapeRegex:function(a){return a.replace(dc,"\\$1")}};var A=function(a,b,c){a[b]!==k&&(a[c]=a[b])},ca=/\[.*?\]$/,V=/\(\)$/,Sa=m.util.escapeRegex,xa=h("<div>")[0],$b=xa.textContent!==k,bc=
-/<.*?>/g,Qa=m.util.throttle,Ub=[],w=Array.prototype,ec=function(a){var b,c,d=m.settings,e=h.map(d,function(a){return a.nTable});if(a){if(a.nTable&&a.oApi)return[a];if(a.nodeName&&"table"===a.nodeName.toLowerCase())return b=h.inArray(a,e),-1!==b?[d[b]]:null;if(a&&"function"===typeof a.settings)return a.settings().toArray();"string"===typeof a?c=h(a):a instanceof h&&(c=a)}else return[];if(c)return c.map(function(){b=h.inArray(this,e);return-1!==b?d[b]:null}).toArray()};t=function(a,b){if(!(this instanceof
-t))return new t(a,b);var c=[],d=function(a){(a=ec(a))&&(c=c.concat(a))};if(h.isArray(a))for(var e=0,f=a.length;e<f;e++)d(a[e]);else d(a);this.context=sa(c);b&&h.merge(this,b);this.selector={rows:null,cols:null,opts:null};t.extend(this,this,Ub)};m.Api=t;h.extend(t.prototype,{any:function(){return 0!==this.count()},concat:w.concat,context:[],count:function(){return this.flatten().length},each:function(a){for(var b=0,c=this.length;b<c;b++)a.call(this,this[b],b,this);return this},eq:function(a){var b=
-this.context;return b.length>a?new t(b[a],this[a]):null},filter:function(a){var b=[];if(w.filter)b=w.filter.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)a.call(this,this[c],c,this)&&b.push(this[c]);return new t(this.context,b)},flatten:function(){var a=[];return new t(this.context,a.concat.apply(a,this.toArray()))},join:w.join,indexOf:w.indexOf||function(a,b){for(var c=b||0,d=this.length;c<d;c++)if(this[c]===a)return c;return-1},iterator:function(a,b,c,d){var e=[],f,g,h,i,n,l=this.context,
-m,p,u=this.selector;"string"===typeof a&&(d=c,c=b,b=a,a=!1);g=0;for(h=l.length;g<h;g++){var s=new t(l[g]);if("table"===b)f=c.call(s,l[g],g),f!==k&&e.push(f);else if("columns"===b||"rows"===b)f=c.call(s,l[g],this[g],g),f!==k&&e.push(f);else if("column"===b||"column-rows"===b||"row"===b||"cell"===b){p=this[g];"column-rows"===b&&(m=Da(l[g],u.opts));i=0;for(n=p.length;i<n;i++)f=p[i],f="cell"===b?c.call(s,l[g],f.row,f.column,g,i):c.call(s,l[g],f,g,i,m),f!==k&&e.push(f)}}return e.length||d?(a=new t(l,a?
-e.concat.apply([],e):e),b=a.selector,b.rows=u.rows,b.cols=u.cols,b.opts=u.opts,a):this},lastIndexOf:w.lastIndexOf||function(a,b){return this.indexOf.apply(this.toArray.reverse(),arguments)},length:0,map:function(a){var b=[];if(w.map)b=w.map.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)b.push(a.call(this,this[c],c));return new t(this.context,b)},pluck:function(a){return this.map(function(b){return b[a]})},pop:w.pop,push:w.push,reduce:w.reduce||function(a,b){return jb(this,a,b,0,this.length,
-1)},reduceRight:w.reduceRight||function(a,b){return jb(this,a,b,this.length-1,-1,-1)},reverse:w.reverse,selector:null,shift:w.shift,slice:function(){return new t(this.context,this)},sort:w.sort,splice:w.splice,toArray:function(){return w.slice.call(this)},to$:function(){return h(this)},toJQuery:function(){return h(this)},unique:function(){return new t(this.context,sa(this))},unshift:w.unshift});t.extend=function(a,b,c){if(c.length&&b&&(b instanceof t||b.__dt_wrapper)){var d,e,f,g=function(a,b,c){return function(){var d=
-b.apply(a,arguments);t.extend(d,d,c.methodExt);return d}};d=0;for(e=c.length;d<e;d++)f=c[d],b[f.name]="function"===typeof f.val?g(a,f.val,f):h.isPlainObject(f.val)?{}:f.val,b[f.name].__dt_wrapper=!0,t.extend(a,b[f.name],f.propExt)}};t.register=p=function(a,b){if(h.isArray(a))for(var c=0,d=a.length;c<d;c++)t.register(a[c],b);else for(var e=a.split("."),f=Ub,g,j,c=0,d=e.length;c<d;c++){g=(j=-1!==e[c].indexOf("()"))?e[c].replace("()",""):e[c];var i;a:{i=0;for(var n=f.length;i<n;i++)if(f[i].name===g){i=
-f[i];break a}i=null}i||(i={name:g,val:{},methodExt:[],propExt:[]},f.push(i));c===d-1?i.val=b:f=j?i.methodExt:i.propExt}};t.registerPlural=u=function(a,b,c){t.register(a,c);t.register(b,function(){var a=c.apply(this,arguments);return a===this?this:a instanceof t?a.length?h.isArray(a[0])?new t(a.context,a[0]):a[0]:k:a})};p("tables()",function(a){var b;if(a){b=t;var c=this.context;if("number"===typeof a)a=[c[a]];else var d=h.map(c,function(a){return a.nTable}),a=h(d).filter(a).map(function(){var a=h.inArray(this,
-d);return c[a]}).toArray();b=new b(a)}else b=this;return b});p("table()",function(a){var a=this.tables(a),b=a.context;return b.length?new t(b[0]):a});u("tables().nodes()","table().node()",function(){return this.iterator("table",function(a){return a.nTable},1)});u("tables().body()","table().body()",function(){return this.iterator("table",function(a){return a.nTBody},1)});u("tables().header()","table().header()",function(){return this.iterator("table",function(a){return a.nTHead},1)});u("tables().footer()",
-"table().footer()",function(){return this.iterator("table",function(a){return a.nTFoot},1)});u("tables().containers()","table().container()",function(){return this.iterator("table",function(a){return a.nTableWrapper},1)});p("draw()",function(a){return this.iterator("table",function(b){"page"===a?O(b):("string"===typeof a&&(a="full-hold"===a?!1:!0),T(b,!1===a))})});p("page()",function(a){return a===k?this.page.info().page:this.iterator("table",function(b){Va(b,a)})});p("page.info()",function(){if(0===
-this.context.length)return k;var a=this.context[0],b=a._iDisplayStart,c=a.oFeatures.bPaginate?a._iDisplayLength:-1,d=a.fnRecordsDisplay(),e=-1===c;return{page:e?0:Math.floor(b/c),pages:e?1:Math.ceil(d/c),start:b,end:a.fnDisplayEnd(),length:c,recordsTotal:a.fnRecordsTotal(),recordsDisplay:d,serverSide:"ssp"===y(a)}});p("page.len()",function(a){return a===k?0!==this.context.length?this.context[0]._iDisplayLength:k:this.iterator("table",function(b){Ta(b,a)})});var Vb=function(a,b,c){if(c){var d=new t(a);
-d.one("draw",function(){c(d.ajax.json())})}if("ssp"==y(a))T(a,b);else{C(a,!0);var e=a.jqXHR;e&&4!==e.readyState&&e.abort();ua(a,[],function(c){pa(a);for(var c=va(a,c),d=0,e=c.length;d<e;d++)N(a,c[d]);T(a,b);C(a,!1)})}};p("ajax.json()",function(){var a=this.context;if(0<a.length)return a[0].json});p("ajax.params()",function(){var a=this.context;if(0<a.length)return a[0].oAjaxData});p("ajax.reload()",function(a,b){return this.iterator("table",function(c){Vb(c,!1===b,a)})});p("ajax.url()",function(a){var b=
-this.context;if(a===k){if(0===b.length)return k;b=b[0];return b.ajax?h.isPlainObject(b.ajax)?b.ajax.url:b.ajax:b.sAjaxSource}return this.iterator("table",function(b){h.isPlainObject(b.ajax)?b.ajax.url=a:b.ajax=a})});p("ajax.url().load()",function(a,b){return this.iterator("table",function(c){Vb(c,!1===b,a)})});var bb=function(a,b,c,d,e){var f=[],g,j,i,n,l,m;i=typeof b;if(!b||"string"===i||"function"===i||b.length===k)b=[b];i=0;for(n=b.length;i<n;i++){j=b[i]&&b[i].split&&!b[i].match(/[\[\(:]/)?b[i].split(","):
-[b[i]];l=0;for(m=j.length;l<m;l++)(g=c("string"===typeof j[l]?h.trim(j[l]):j[l]))&&g.length&&(f=f.concat(g))}a=x.selector[a];if(a.length){i=0;for(n=a.length;i<n;i++)f=a[i](d,e,f)}return sa(f)},cb=function(a){a||(a={});a.filter&&a.search===k&&(a.search=a.filter);return h.extend({search:"none",order:"current",page:"all"},a)},db=function(a){for(var b=0,c=a.length;b<c;b++)if(0<a[b].length)return a[0]=a[b],a[0].length=1,a.length=1,a.context=[a.context[b]],a;a.length=0;return a},Da=function(a,b){var c,
-d,e,f=[],g=a.aiDisplay;c=a.aiDisplayMaster;var j=b.search;d=b.order;e=b.page;if("ssp"==y(a))return"removed"===j?[]:X(0,c.length);if("current"==e){c=a._iDisplayStart;for(d=a.fnDisplayEnd();c<d;c++)f.push(g[c])}else if("current"==d||"applied"==d)f="none"==j?c.slice():"applied"==j?g.slice():h.map(c,function(a){return-1===h.inArray(a,g)?a:null});else if("index"==d||"original"==d){c=0;for(d=a.aoData.length;c<d;c++)"none"==j?f.push(c):(e=h.inArray(c,g),(-1===e&&"removed"==j||0<=e&&"applied"==j)&&f.push(c))}return f};
-p("rows()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=cb(b),c=this.iterator("table",function(c){var e=b,f;return bb("row",a,function(a){var b=Qb(a);if(b!==null&&!e)return[b];f||(f=Da(c,e));if(b!==null&&h.inArray(b,f)!==-1)return[b];if(a===null||a===k||a==="")return f;if(typeof a==="function")return h.map(f,function(b){var e=c.aoData[b];return a(b,e._aData,e.nTr)?b:null});b=Tb(ja(c.aoData,f,"nTr"));if(a.nodeName){if(a._DT_RowIndex!==k)return[a._DT_RowIndex];if(a._DT_CellIndex)return[a._DT_CellIndex.row];
-b=h(a).closest("*[data-dt-row]");return b.length?[b.data("dt-row")]:[]}if(typeof a==="string"&&a.charAt(0)==="#"){var i=c.aIds[a.replace(/^#/,"")];if(i!==k)return[i.idx]}return h(b).filter(a).map(function(){return this._DT_RowIndex}).toArray()},c,e)},1);c.selector.rows=a;c.selector.opts=b;return c});p("rows().nodes()",function(){return this.iterator("row",function(a,b){return a.aoData[b].nTr||k},1)});p("rows().data()",function(){return this.iterator(!0,"rows",function(a,b){return ja(a.aoData,b,"_aData")},
-1)});u("rows().cache()","row().cache()",function(a){return this.iterator("row",function(b,c){var d=b.aoData[c];return"search"===a?d._aFilterData:d._aSortData},1)});u("rows().invalidate()","row().invalidate()",function(a){return this.iterator("row",function(b,c){da(b,c,a)})});u("rows().indexes()","row().index()",function(){return this.iterator("row",function(a,b){return b},1)});u("rows().ids()","row().id()",function(a){for(var b=[],c=this.context,d=0,e=c.length;d<e;d++)for(var f=0,g=this[d].length;f<
-g;f++){var h=c[d].rowIdFn(c[d].aoData[this[d][f]]._aData);b.push((!0===a?"#":"")+h)}return new t(c,b)});u("rows().remove()","row().remove()",function(){var a=this;this.iterator("row",function(b,c,d){var e=b.aoData,f=e[c],g,h,i,n,l;e.splice(c,1);g=0;for(h=e.length;g<h;g++)if(i=e[g],l=i.anCells,null!==i.nTr&&(i.nTr._DT_RowIndex=g),null!==l){i=0;for(n=l.length;i<n;i++)l[i]._DT_CellIndex.row=g}qa(b.aiDisplayMaster,c);qa(b.aiDisplay,c);qa(a[d],c,!1);Ua(b);c=b.rowIdFn(f._aData);c!==k&&delete b.aIds[c]});
-this.iterator("table",function(a){for(var c=0,d=a.aoData.length;c<d;c++)a.aoData[c].idx=c});return this});p("rows.add()",function(a){var b=this.iterator("table",function(b){var c,f,g,h=[];f=0;for(g=a.length;f<g;f++)c=a[f],c.nodeName&&"TR"===c.nodeName.toUpperCase()?h.push(oa(b,c)[0]):h.push(N(b,c));return h},1),c=this.rows(-1);c.pop();h.merge(c,b);return c});p("row()",function(a,b){return db(this.rows(a,b))});p("row().data()",function(a){var b=this.context;if(a===k)return b.length&&this.length?b[0].aoData[this[0]]._aData:
-k;b[0].aoData[this[0]]._aData=a;da(b[0],this[0],"data");return this});p("row().node()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]].nTr||null:null});p("row.add()",function(a){a instanceof h&&a.length&&(a=a[0]);var b=this.iterator("table",function(b){return a.nodeName&&"TR"===a.nodeName.toUpperCase()?oa(b,a)[0]:N(b,a)});return this.row(b[0])});var eb=function(a,b){var c=a.context;if(c.length&&(c=c[0].aoData[b!==k?b:a[0]])&&c._details)c._details.remove(),c._detailsShow=
-k,c._details=k},Wb=function(a,b){var c=a.context;if(c.length&&a.length){var d=c[0].aoData[a[0]];if(d._details){(d._detailsShow=b)?d._details.insertAfter(d.nTr):d._details.detach();var e=c[0],f=new t(e),g=e.aoData;f.off("draw.dt.DT_details column-visibility.dt.DT_details destroy.dt.DT_details");0<D(g,"_details").length&&(f.on("draw.dt.DT_details",function(a,b){e===b&&f.rows({page:"current"}).eq(0).each(function(a){a=g[a];a._detailsShow&&a._details.insertAfter(a.nTr)})}),f.on("column-visibility.dt.DT_details",
-function(a,b){if(e===b)for(var c,d=ba(b),f=0,h=g.length;f<h;f++)c=g[f],c._details&&c._details.children("td[colspan]").attr("colspan",d)}),f.on("destroy.dt.DT_details",function(a,b){if(e===b)for(var c=0,d=g.length;c<d;c++)g[c]._details&&eb(f,c)}))}}};p("row().child()",function(a,b){var c=this.context;if(a===k)return c.length&&this.length?c[0].aoData[this[0]]._details:k;if(!0===a)this.child.show();else if(!1===a)eb(this);else if(c.length&&this.length){var d=c[0],c=c[0].aoData[this[0]],e=[],f=function(a,
-b){if(h.isArray(a)||a instanceof h)for(var c=0,k=a.length;c<k;c++)f(a[c],b);else a.nodeName&&"tr"===a.nodeName.toLowerCase()?e.push(a):(c=h("<tr><td/></tr>").addClass(b),h("td",c).addClass(b).html(a)[0].colSpan=ba(d),e.push(c[0]))};f(a,b);c._details&&c._details.detach();c._details=h(e);c._detailsShow&&c._details.insertAfter(c.nTr)}return this});p(["row().child.show()","row().child().show()"],function(){Wb(this,!0);return this});p(["row().child.hide()","row().child().hide()"],function(){Wb(this,!1);
-return this});p(["row().child.remove()","row().child().remove()"],function(){eb(this);return this});p("row().child.isShown()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var fc=/^([^:]+):(name|visIdx|visible)$/,Xb=function(a,b,c,d,e){for(var c=[],d=0,f=e.length;d<f;d++)c.push(B(a,e[d],b));return c};p("columns()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=cb(b),c=this.iterator("table",function(c){var e=a,f=b,g=c.aoColumns,
-j=D(g,"sName"),i=D(g,"nTh");return bb("column",e,function(a){var b=Qb(a);if(a==="")return X(g.length);if(b!==null)return[b>=0?b:g.length+b];if(typeof a==="function"){var e=Da(c,f);return h.map(g,function(b,f){return a(f,Xb(c,f,0,0,e),i[f])?f:null})}var k=typeof a==="string"?a.match(fc):"";if(k)switch(k[2]){case "visIdx":case "visible":b=parseInt(k[1],10);if(b<0){var m=h.map(g,function(a,b){return a.bVisible?b:null});return[m[m.length+b]]}return[$(c,b)];case "name":return h.map(j,function(a,b){return a===
-k[1]?b:null});default:return[]}if(a.nodeName&&a._DT_CellIndex)return[a._DT_CellIndex.column];b=h(i).filter(a).map(function(){return h.inArray(this,i)}).toArray();if(b.length||!a.nodeName)return b;b=h(a).closest("*[data-dt-column]");return b.length?[b.data("dt-column")]:[]},c,f)},1);c.selector.cols=a;c.selector.opts=b;return c});u("columns().header()","column().header()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTh},1)});u("columns().footer()","column().footer()",
-function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTf},1)});u("columns().data()","column().data()",function(){return this.iterator("column-rows",Xb,1)});u("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData},1)});u("columns().cache()","column().cache()",function(a){return this.iterator("column-rows",function(b,c,d,e,f){return ja(b.aoData,f,"search"===a?"_aFilterData":"_aSortData",c)},1)});u("columns().nodes()",
-"column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,d,e){return ja(a.aoData,e,"anCells",b)},1)});u("columns().visible()","column().visible()",function(a,b){var c=this.iterator("column",function(b,c){if(a===k)return b.aoColumns[c].bVisible;var f=b.aoColumns,g=f[c],j=b.aoData,i,n,l;if(a!==k&&g.bVisible!==a){if(a){var m=h.inArray(!0,D(f,"bVisible"),c+1);i=0;for(n=j.length;i<n;i++)l=j[i].nTr,f=j[i].anCells,l&&l.insertBefore(f[c],f[m]||null)}else h(D(b.aoData,"anCells",c)).detach();
-g.bVisible=a;fa(b,b.aoHeader);fa(b,b.aoFooter);za(b)}});a!==k&&(this.iterator("column",function(c,e){s(c,null,"column-visibility",[c,e,a,b])}),(b===k||b)&&this.columns.adjust());return c});u("columns().indexes()","column().index()",function(a){return this.iterator("column",function(b,c){return"visible"===a?aa(b,c):c},1)});p("columns.adjust()",function(){return this.iterator("table",function(a){Z(a)},1)});p("column.index()",function(a,b){if(0!==this.context.length){var c=this.context[0];if("fromVisible"===
-a||"toData"===a)return $(c,b);if("fromData"===a||"toVisible"===a)return aa(c,b)}});p("column()",function(a,b){return db(this.columns(a,b))});p("cells()",function(a,b,c){h.isPlainObject(a)&&(a.row===k?(c=a,a=null):(c=b,b=null));h.isPlainObject(b)&&(c=b,b=null);if(null===b||b===k)return this.iterator("table",function(b){var d=a,e=cb(c),f=b.aoData,g=Da(b,e),j=Tb(ja(f,g,"anCells")),i=h([].concat.apply([],j)),l,n=b.aoColumns.length,m,p,u,t,s,v;return bb("cell",d,function(a){var c=typeof a==="function";
-if(a===null||a===k||c){m=[];p=0;for(u=g.length;p<u;p++){l=g[p];for(t=0;t<n;t++){s={row:l,column:t};if(c){v=f[l];a(s,B(b,l,t),v.anCells?v.anCells[t]:null)&&m.push(s)}else m.push(s)}}return m}if(h.isPlainObject(a))return[a];c=i.filter(a).map(function(a,b){return{row:b._DT_CellIndex.row,column:b._DT_CellIndex.column}}).toArray();if(c.length||!a.nodeName)return c;v=h(a).closest("*[data-dt-row]");return v.length?[{row:v.data("dt-row"),column:v.data("dt-column")}]:[]},b,e)});var d=this.columns(b,c),e=this.rows(a,
-c),f,g,j,i,n,l=this.iterator("table",function(a,b){f=[];g=0;for(j=e[b].length;g<j;g++){i=0;for(n=d[b].length;i<n;i++)f.push({row:e[b][g],column:d[b][i]})}return f},1);h.extend(l.selector,{cols:b,rows:a,opts:c});return l});u("cells().nodes()","cell().node()",function(){return this.iterator("cell",function(a,b,c){return(a=a.aoData[b])&&a.anCells?a.anCells[c]:k},1)});p("cells().data()",function(){return this.iterator("cell",function(a,b,c){return B(a,b,c)},1)});u("cells().cache()","cell().cache()",function(a){a=
-"search"===a?"_aFilterData":"_aSortData";return this.iterator("cell",function(b,c,d){return b.aoData[c][a][d]},1)});u("cells().render()","cell().render()",function(a){return this.iterator("cell",function(b,c,d){return B(b,c,d,a)},1)});u("cells().indexes()","cell().index()",function(){return this.iterator("cell",function(a,b,c){return{row:b,column:c,columnVisible:aa(a,c)}},1)});u("cells().invalidate()","cell().invalidate()",function(a){return this.iterator("cell",function(b,c,d){da(b,c,a,d)})});p("cell()",
-function(a,b,c){return db(this.cells(a,b,c))});p("cell().data()",function(a){var b=this.context,c=this[0];if(a===k)return b.length&&c.length?B(b[0],c[0].row,c[0].column):k;lb(b[0],c[0].row,c[0].column,a);da(b[0],c[0].row,"data",c[0].column);return this});p("order()",function(a,b){var c=this.context;if(a===k)return 0!==c.length?c[0].aaSorting:k;"number"===typeof a?a=[[a,b]]:a.length&&!h.isArray(a[0])&&(a=Array.prototype.slice.call(arguments));return this.iterator("table",function(b){b.aaSorting=a.slice()})});
-p("order.listener()",function(a,b,c){return this.iterator("table",function(d){Oa(d,a,b,c)})});p("order.fixed()",function(a){if(!a){var b=this.context,b=b.length?b[0].aaSortingFixed:k;return h.isArray(b)?{pre:b}:b}return this.iterator("table",function(b){b.aaSortingFixed=h.extend(!0,{},a)})});p(["columns().order()","column().order()"],function(a){var b=this;return this.iterator("table",function(c,d){var e=[];h.each(b[d],function(b,c){e.push([c,a])});c.aaSorting=e})});p("search()",function(a,b,c,d){var e=
-this.context;return a===k?0!==e.length?e[0].oPreviousSearch.sSearch:k:this.iterator("table",function(e){e.oFeatures.bFilter&&ga(e,h.extend({},e.oPreviousSearch,{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===d?!0:d}),1)})});u("columns().search()","column().search()",function(a,b,c,d){return this.iterator("column",function(e,f){var g=e.aoPreSearchCols;if(a===k)return g[f].sSearch;e.oFeatures.bFilter&&(h.extend(g[f],{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?
-!0:c,bCaseInsensitive:null===d?!0:d}),ga(e,e.oPreviousSearch,1))})});p("state()",function(){return this.context.length?this.context[0].oSavedState:null});p("state.clear()",function(){return this.iterator("table",function(a){a.fnStateSaveCallback.call(a.oInstance,a,{})})});p("state.loaded()",function(){return this.context.length?this.context[0].oLoadedState:null});p("state.save()",function(){return this.iterator("table",function(a){za(a)})});m.versionCheck=m.fnVersionCheck=function(a){for(var b=m.version.split("."),
-a=a.split("."),c,d,e=0,f=a.length;e<f;e++)if(c=parseInt(b[e],10)||0,d=parseInt(a[e],10)||0,c!==d)return c>d;return!0};m.isDataTable=m.fnIsDataTable=function(a){var b=h(a).get(0),c=!1;if(a instanceof m.Api)return!0;h.each(m.settings,function(a,e){var f=e.nScrollHead?h("table",e.nScrollHead)[0]:null,g=e.nScrollFoot?h("table",e.nScrollFoot)[0]:null;if(e.nTable===b||f===b||g===b)c=!0});return c};m.tables=m.fnTables=function(a){var b=!1;h.isPlainObject(a)&&(b=a.api,a=a.visible);var c=h.map(m.settings,
-function(b){if(!a||a&&h(b.nTable).is(":visible"))return b.nTable});return b?new t(c):c};m.camelToHungarian=J;p("$()",function(a,b){var c=this.rows(b).nodes(),c=h(c);return h([].concat(c.filter(a).toArray(),c.find(a).toArray()))});h.each(["on","one","off"],function(a,b){p(b+"()",function(){var a=Array.prototype.slice.call(arguments);a[0]=h.map(a[0].split(/\s/),function(a){return!a.match(/\.dt\b/)?a+".dt":a}).join(" ");var d=h(this.tables().nodes());d[b].apply(d,a);return this})});p("clear()",function(){return this.iterator("table",
-function(a){pa(a)})});p("settings()",function(){return new t(this.context,this.context)});p("init()",function(){var a=this.context;return a.length?a[0].oInit:null});p("data()",function(){return this.iterator("table",function(a){return D(a.aoData,"_aData")}).flatten()});p("destroy()",function(a){a=a||!1;return this.iterator("table",function(b){var c=b.nTableWrapper.parentNode,d=b.oClasses,e=b.nTable,f=b.nTBody,g=b.nTHead,j=b.nTFoot,i=h(e),f=h(f),k=h(b.nTableWrapper),l=h.map(b.aoData,function(a){return a.nTr}),
-p;b.bDestroying=!0;s(b,"aoDestroyCallback","destroy",[b]);a||(new t(b)).columns().visible(!0);k.off(".DT").find(":not(tbody *)").off(".DT");h(E).off(".DT-"+b.sInstance);e!=g.parentNode&&(i.children("thead").detach(),i.append(g));j&&e!=j.parentNode&&(i.children("tfoot").detach(),i.append(j));b.aaSorting=[];b.aaSortingFixed=[];ya(b);h(l).removeClass(b.asStripeClasses.join(" "));h("th, td",g).removeClass(d.sSortable+" "+d.sSortableAsc+" "+d.sSortableDesc+" "+d.sSortableNone);b.bJUI&&(h("th span."+d.sSortIcon+
-", td span."+d.sSortIcon,g).detach(),h("th, td",g).each(function(){var a=h("div."+d.sSortJUIWrapper,this);h(this).append(a.contents());a.detach()}));f.children().detach();f.append(l);g=a?"remove":"detach";i[g]();k[g]();!a&&c&&(c.insertBefore(e,b.nTableReinsertBefore),i.css("width",b.sDestroyWidth).removeClass(d.sTable),(p=b.asDestroyStripes.length)&&f.children().each(function(a){h(this).addClass(b.asDestroyStripes[a%p])}));c=h.inArray(b,m.settings);-1!==c&&m.settings.splice(c,1)})});h.each(["column",
-"row","cell"],function(a,b){p(b+"s().every()",function(a){var d=this.selector.opts,e=this;return this.iterator(b,function(f,g,h,i,m){a.call(e[b](g,"cell"===b?h:d,"cell"===b?d:k),g,h,i,m)})})});p("i18n()",function(a,b,c){var d=this.context[0],a=R(a)(d.oLanguage);a===k&&(a=b);c!==k&&h.isPlainObject(a)&&(a=a[c]!==k?a[c]:a._);return a.replace("%d",c)});m.version="1.10.15";m.settings=[];m.models={};m.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};m.models.oRow={nTr:null,anCells:null,
-_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null,idx:-1};m.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};m.defaults=
-{aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bJQueryUI:!1,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,
-this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+a.sInstance+"_"+location.pathname))}catch(b){}},fnStateLoadParams:null,fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+
-"_"+location.pathname,JSON.stringify(b))}catch(c){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",
-sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},m.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null,rowId:"DT_RowId"};
-Y(m.defaults);m.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};Y(m.defaults.column);m.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,
-bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1,bBounding:!1,barWidth:0},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aIds:{},aoColumns:[],aoHeader:[],aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],
-aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:!0,jqXHR:null,json:k,oAjaxData:k,fnServerData:null,
-aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,bJUI:null,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==y(this)?1*this._iRecordsTotal:this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==y(this)?1*this._iRecordsDisplay:this.aiDisplay.length},fnDisplayEnd:function(){var a=
-this._iDisplayLength,b=this._iDisplayStart,c=b+a,d=this.aiDisplay.length,e=this.oFeatures,f=e.bPaginate;return e.bServerSide?!1===f||-1===a?b+d:Math.min(b+a,this._iRecordsDisplay):!f||c>d||-1===a?d:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{},rowIdFn:null,rowId:null};m.ext=x={buttons:{},classes:{},builder:"-source-",errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},
-header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:m.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:m.version};h.extend(x,{afnFiltering:x.search,aTypes:x.type.detect,ofnSearch:x.type.search,oSort:x.type.order,afnSortData:x.order,aoFeatures:x.feature,oApi:x.internal,oStdClasses:x.classes,oPagination:x.pager});h.extend(m.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",
-sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",
-sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",sJUIHeader:"",sJUIFooter:""});var Ea="",Ea="",G=Ea+"ui-state-default",ka=Ea+"css_right ui-icon ui-icon-",Yb=Ea+"fg-toolbar ui-toolbar ui-widget-header ui-helper-clearfix";h.extend(m.ext.oJUIClasses,
-m.ext.classes,{sPageButton:"fg-button ui-button "+G,sPageButtonActive:"ui-state-disabled",sPageButtonDisabled:"ui-state-disabled",sPaging:"dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",sSortAsc:G+" sorting_asc",sSortDesc:G+" sorting_desc",sSortable:G+" sorting",sSortableAsc:G+" sorting_asc_disabled",sSortableDesc:G+" sorting_desc_disabled",sSortableNone:G+" sorting_disabled",sSortJUIAsc:ka+"triangle-1-n",sSortJUIDesc:ka+"triangle-1-s",sSortJUI:ka+"carat-2-n-s",
-sSortJUIAscAllowed:ka+"carat-1-n",sSortJUIDescAllowed:ka+"carat-1-s",sSortJUIWrapper:"DataTables_sort_wrapper",sSortIcon:"DataTables_sort_icon",sScrollHead:"dataTables_scrollHead "+G,sScrollFoot:"dataTables_scrollFoot "+G,sHeaderTH:G,sFooterTH:G,sJUIHeader:Yb+" ui-corner-tl ui-corner-tr",sJUIFooter:Yb+" ui-corner-bl ui-corner-br"});var Nb=m.ext.pager;h.extend(Nb,{simple:function(){return["previous","next"]},full:function(){return["first","previous","next","last"]},numbers:function(a,b){return[ia(a,
-b)]},simple_numbers:function(a,b){return["previous",ia(a,b),"next"]},full_numbers:function(a,b){return["first","previous",ia(a,b),"next","last"]},first_last_numbers:function(a,b){return["first",ia(a,b),"last"]},_numbers:ia,numbers_length:7});h.extend(!0,m.ext.renderer,{pageButton:{_:function(a,b,c,d,e,f){var g=a.oClasses,j=a.oLanguage.oPaginate,i=a.oLanguage.oAria.paginate||{},m,l,p=0,r=function(b,d){var k,t,u,s,v=function(b){Va(a,b.data.action,true)};k=0;for(t=d.length;k<t;k++){s=d[k];if(h.isArray(s)){u=
-h("<"+(s.DT_el||"div")+"/>").appendTo(b);r(u,s)}else{m=null;l="";switch(s){case "ellipsis":b.append('<span class="ellipsis">…</span>');break;case "first":m=j.sFirst;l=s+(e>0?"":" "+g.sPageButtonDisabled);break;case "previous":m=j.sPrevious;l=s+(e>0?"":" "+g.sPageButtonDisabled);break;case "next":m=j.sNext;l=s+(e<f-1?"":" "+g.sPageButtonDisabled);break;case "last":m=j.sLast;l=s+(e<f-1?"":" "+g.sPageButtonDisabled);break;default:m=s+1;l=e===s?g.sPageButtonActive:""}if(m!==null){u=h("<a>",{"class":g.sPageButton+
-" "+l,"aria-controls":a.sTableId,"aria-label":i[s],"data-dt-idx":p,tabindex:a.iTabIndex,id:c===0&&typeof s==="string"?a.sTableId+"_"+s:null}).html(m).appendTo(b);Ya(u,{action:s},v);p++}}}},t;try{t=h(b).find(H.activeElement).data("dt-idx")}catch(u){}r(h(b).empty(),d);t!==k&&h(b).find("[data-dt-idx="+t+"]").focus()}}});h.extend(m.ext.type.detect,[function(a,b){var c=b.oLanguage.sDecimal;return ab(a,c)?"num"+c:null},function(a){if(a&&!(a instanceof Date)&&!cc.test(a))return null;var b=Date.parse(a);
-return null!==b&&!isNaN(b)||M(a)?"date":null},function(a,b){var c=b.oLanguage.sDecimal;return ab(a,c,!0)?"num-fmt"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Sb(a,c)?"html-num"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Sb(a,c,!0)?"html-num-fmt"+c:null},function(a){return M(a)||"string"===typeof a&&-1!==a.indexOf("<")?"html":null}]);h.extend(m.ext.type.search,{html:function(a){return M(a)?a:"string"===typeof a?a.replace(Pb," ").replace(Ca,""):""},string:function(a){return M(a)?
-a:"string"===typeof a?a.replace(Pb," "):a}});var Ba=function(a,b,c,d){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=Rb(a,b));a.replace&&(c&&(a=a.replace(c,"")),d&&(a=a.replace(d,"")));return 1*a};h.extend(x.type.order,{"date-pre":function(a){return Date.parse(a)||-Infinity},"html-pre":function(a){return M(a)?"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():a+""},"string-pre":function(a){return M(a)?"":"string"===typeof a?a.toLowerCase():!a.toString?"":a.toString()},"string-asc":function(a,b){return a<
-b?-1:a>b?1:0},"string-desc":function(a,b){return a<b?1:a>b?-1:0}});fb("");h.extend(!0,m.ext.renderer,{header:{_:function(a,b,c,d){h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(c.sSortingClass+" "+d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:c.sSortingClass)}})},jqueryui:function(a,b,c,d){h("<div/>").addClass(d.sSortJUIWrapper).append(b.contents()).append(h("<span/>").addClass(d.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);
-h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:c.sSortingClass);b.find("span."+d.sSortIcon).removeClass(d.sSortJUIAsc+" "+d.sSortJUIDesc+" "+d.sSortJUI+" "+d.sSortJUIAscAllowed+" "+d.sSortJUIDescAllowed).addClass(h[e]=="asc"?d.sSortJUIAsc:h[e]=="desc"?d.sSortJUIDesc:c.sSortingClassJUI)}})}}});var Zb=function(a){return"string"===typeof a?a.replace(/</g,"<").replace(/>/g,">").replace(/"/g,
-"""):a};m.render={number:function(a,b,c,d,e){return{display:function(f){if("number"!==typeof f&&"string"!==typeof f)return f;var g=0>f?"-":"",h=parseFloat(f);if(isNaN(h))return Zb(f);h=h.toFixed(c);f=Math.abs(h);h=parseInt(f,10);f=c?b+(f-h).toFixed(c).substring(2):"";return g+(d||"")+h.toString().replace(/\B(?=(\d{3})+(?!\d))/g,a)+f+(e||"")}}},text:function(){return{display:Zb}}};h.extend(m.ext.internal,{_fnExternApiFunc:Ob,_fnBuildAjax:ua,_fnAjaxUpdate:nb,_fnAjaxParameters:wb,_fnAjaxUpdateDraw:xb,
-_fnAjaxDataSrc:va,_fnAddColumn:Ga,_fnColumnOptions:la,_fnAdjustColumnSizing:Z,_fnVisibleToColumnIndex:$,_fnColumnIndexToVisible:aa,_fnVisbleColumns:ba,_fnGetColumns:na,_fnColumnTypes:Ia,_fnApplyColumnDefs:kb,_fnHungarianMap:Y,_fnCamelToHungarian:J,_fnLanguageCompat:Fa,_fnBrowserDetect:ib,_fnAddData:N,_fnAddTr:oa,_fnNodeToDataIndex:function(a,b){return b._DT_RowIndex!==k?b._DT_RowIndex:null},_fnNodeToColumnIndex:function(a,b,c){return h.inArray(c,a.aoData[b].anCells)},_fnGetCellData:B,_fnSetCellData:lb,
-_fnSplitObjNotation:La,_fnGetObjectDataFn:R,_fnSetObjectDataFn:S,_fnGetDataMaster:Ma,_fnClearTable:pa,_fnDeleteIndex:qa,_fnInvalidate:da,_fnGetRowElements:Ka,_fnCreateTr:Ja,_fnBuildHead:mb,_fnDrawHead:fa,_fnDraw:O,_fnReDraw:T,_fnAddOptionsHtml:pb,_fnDetectHeader:ea,_fnGetUniqueThs:ta,_fnFeatureHtmlFilter:rb,_fnFilterComplete:ga,_fnFilterCustom:Ab,_fnFilterColumn:zb,_fnFilter:yb,_fnFilterCreateSearch:Ra,_fnEscapeRegex:Sa,_fnFilterData:Bb,_fnFeatureHtmlInfo:ub,_fnUpdateInfo:Eb,_fnInfoMacros:Fb,_fnInitialise:ha,
-_fnInitComplete:wa,_fnLengthChange:Ta,_fnFeatureHtmlLength:qb,_fnFeatureHtmlPaginate:vb,_fnPageChange:Va,_fnFeatureHtmlProcessing:sb,_fnProcessingDisplay:C,_fnFeatureHtmlTable:tb,_fnScrollDraw:ma,_fnApplyToChildren:I,_fnCalculateColumnWidths:Ha,_fnThrottle:Qa,_fnConvertToWidth:Gb,_fnGetWidestNode:Hb,_fnGetMaxLenString:Ib,_fnStringToCss:v,_fnSortFlatten:W,_fnSort:ob,_fnSortAria:Kb,_fnSortListener:Xa,_fnSortAttachListener:Oa,_fnSortingClasses:ya,_fnSortData:Jb,_fnSaveState:za,_fnLoadState:Lb,_fnSettingsFromNode:Aa,
-_fnLog:K,_fnMap:F,_fnBindAction:Ya,_fnCallbackReg:z,_fnCallbackFire:s,_fnLengthOverflow:Ua,_fnRenderer:Pa,_fnDataSource:y,_fnRowAttributes:Na,_fnCalculateEnd:function(){}});h.fn.dataTable=m;m.$=h;h.fn.dataTableSettings=m.settings;h.fn.dataTableExt=m.ext;h.fn.DataTable=function(a){return h(this).dataTable(a).api()};h.each(m,function(a,b){h.fn.DataTable[a]=b});return h.fn.dataTable});
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/livy-mini-logo.png
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/livy-mini-logo.png b/server/src/main/resources/com/cloudera/livy/server/ui/static/livy-mini-logo.png
deleted file mode 100644
index 49606b4..0000000
Binary files a/server/src/main/resources/com/cloudera/livy/server/ui/static/livy-mini-logo.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/livy-ui.css
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/livy-ui.css b/server/src/main/resources/com/cloudera/livy/server/ui/static/livy-ui.css
deleted file mode 100644
index aadb256..0000000
--- a/server/src/main/resources/com/cloudera/livy/server/ui/static/livy-ui.css
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-body {
- padding-top: 20px;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/sessions-table.html
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/sessions-table.html b/server/src/main/resources/com/cloudera/livy/server/ui/static/sessions-table.html
deleted file mode 100644
index a50d328..0000000
--- a/server/src/main/resources/com/cloudera/livy/server/ui/static/sessions-table.html
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<h4 id="interactive-sessions-header" class="sessions-template">Interactive Sessions</h4>
-
-<table id="interactive-sessions-table"
- class="table table-striped sessions-table sessions-template">
- <thead class="sessions-table-head">
- <tr>
- <th>Session Id</th>
- <th>
- <span data-toggle="tooltip"
- title="Spark Application Id for this session.
- If available, links to Spark Application Web UI">
- Application Id
- </span>
- </th>
- <th>
- <span data-toggle="tooltip" title="Remote user who submitted this session">
- Owner
- </span>
- </th>
- <th>
- <span data-toggle="tooltip" title="User to impersonate when running">
- Proxy User
- </span>
- </th>
- <th>
- <span data-toggle="tooltip"
- title="Session kind (spark, pyspark, pyspark3, or sparkr)">
- Session Kind
- </span>
- </th>
- <th>
- <span data-toggle="tooltip"
- title="Session State (not_started, starting, idle, busy,
- shutting_down, error, dead, success)">
- State
- </span>
- </th>
- </tr>
- </thead>
- <tbody class="sessions-table-body">
- </tbody>
-</table>
\ No newline at end of file
[14/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/all-sessions.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/all-sessions.js b/server/src/main/resources/org/apache/livy/server/ui/static/all-sessions.js
new file mode 100644
index 0000000..4fe3f8f
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/all-sessions.js
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+function appIdLink(session) {
+ var appUiUrl = session.appInfo.sparkUiUrl;
+ if (appUiUrl != null) {
+ return '<a href="' + appUiUrl + '">' + session.appId + "</a>";
+ } else {
+ return session.appId;
+ }
+}
+
+function tdWrap(str) {
+ return "<td>" + str + "</td>";
+}
+
+function loadSessionsTable(sessions) {
+ $.each(sessions, function(index, session) {
+ $("#interactive-sessions .sessions-table-body").append(
+ "<tr>" +
+ tdWrap(session.id) +
+ tdWrap(appIdLink(session)) +
+ tdWrap(session.owner) +
+ tdWrap(session.proxyUser) +
+ tdWrap(session.kind) +
+ tdWrap(session.state) +
+ "</tr>"
+ );
+ });
+}
+
+function loadBatchesTable(sessions) {
+ $.each(sessions, function(index, session) {
+ $("#batches .sessions-table-body").append(
+ "<tr>" +
+ tdWrap(session.id) +
+ tdWrap(appIdLink(session)) +
+ tdWrap(session.state) +
+ "</tr>"
+ );
+ });
+}
+
+var numSessions = 0;
+var numBatches = 0;
+
+$(document).ready(function () {
+ $.extend( $.fn.dataTable.defaults, {
+ stateSave: true,
+ });
+
+ var sessionsReq = $.getJSON(location.origin + "/sessions", function(response) {
+ if (response && response.total > 0) {
+ $("#interactive-sessions").load("/static/sessions-table.html .sessions-template", function() {
+ loadSessionsTable(response.sessions);
+ $("#interactive-sessions-table").DataTable();
+ $('#interactive-sessions [data-toggle="tooltip"]').tooltip();
+ });
+ }
+ numSessions = response.total;
+ });
+
+ var batchesReq = $.getJSON(location.origin + "/batches", function(response) {
+ if (response && response.total > 0) {
+ $("#batches").load("/static/batches-table.html .sessions-template", function() {
+ loadBatchesTable(response.sessions);
+ $("#batches-table").DataTable();
+ $('#batches [data-toggle="tooltip"]').tooltip();
+ });
+ }
+ numBatches = response.total;
+ });
+
+ $.when(sessionsReq, batchesReq).done(function () {
+ if (numSessions + numBatches == 0) {
+ $("#all-sessions").append('<h4>No Sessions or Batches have been created yet.</h4>');
+ }
+ });
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/batches-table.html
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/batches-table.html b/server/src/main/resources/org/apache/livy/server/ui/static/batches-table.html
new file mode 100644
index 0000000..e0b3213
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/batches-table.html
@@ -0,0 +1,42 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h4 id="batches-header" class="sessions-template">Batch Sessions</h4>
+
+<table id="batches-table" class="table table-striped sessions-table sessions-template">
+ <thead class="sessions-table-head">
+ <tr>
+ <th>Batch Id</th>
+ <th>
+ <span data-toggle="tooltip"
+ title="Spark Application Id for this session.
+ If available, links to Spark Application Web UI">
+ Application Id
+ </span>
+ </th>
+ <th>
+ <span data-toggle="tooltip"
+ title="Session State (not_started, starting, idle, busy,
+ shutting_down, error, dead, success)">
+ State
+ </span>
+ </th>
+ </tr>
+ </thead>
+ <tbody class="sessions-table-body">
+ </tbody>
+</table>
\ No newline at end of file
[29/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index e32ca78..e15c470 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -20,14 +20,14 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-main</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>livy-integration-test</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/main/scala/com/cloudera/livy/test/framework/BaseIntegrationTestSuite.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/com/cloudera/livy/test/framework/BaseIntegrationTestSuite.scala b/integration-test/src/main/scala/com/cloudera/livy/test/framework/BaseIntegrationTestSuite.scala
deleted file mode 100644
index 039ee0a..0000000
--- a/integration-test/src/main/scala/com/cloudera/livy/test/framework/BaseIntegrationTestSuite.scala
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.framework
-
-import java.io.File
-import java.util.UUID
-
-import scala.concurrent._
-import scala.concurrent.duration._
-import scala.language.postfixOps
-import scala.util.control.NonFatal
-
-import com.ning.http.client.AsyncHttpClient
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.yarn.util.ConverterUtils
-import org.scalatest._
-
-abstract class BaseIntegrationTestSuite extends FunSuite with Matchers with BeforeAndAfterAll {
- import scala.concurrent.ExecutionContext.Implicits.global
-
- var cluster: Cluster = _
- var httpClient: AsyncHttpClient = _
- var livyClient: LivyRestClient = _
-
- protected def livyEndpoint: String = cluster.livyEndpoint
-
- protected val testLib = sys.props("java.class.path")
- .split(File.pathSeparator)
- .find(new File(_).getName().startsWith("livy-test-lib-"))
- .getOrElse(throw new Exception(s"Cannot find test lib in ${sys.props("java.class.path")}"))
-
- protected def getYarnLog(appId: String): String = {
- require(appId != null, "appId shouldn't be null")
-
- val appReport = cluster.yarnClient.getApplicationReport(ConverterUtils.toApplicationId(appId))
- assert(appReport != null, "appReport shouldn't be null")
-
- appReport.getDiagnostics()
- }
-
- protected def restartLivy(): Unit = {
- val f = future {
- cluster.stopLivy()
- cluster.runLivy()
- }
- Await.result(f, 3 minutes)
- }
-
- /** Uploads a file to HDFS and returns just its path. */
- protected def uploadToHdfs(file: File): String = {
- val hdfsPath = new Path(cluster.hdfsScratchDir(),
- UUID.randomUUID().toString() + "-" + file.getName())
- cluster.fs.copyFromLocalFile(new Path(file.toURI()), hdfsPath)
- hdfsPath.toUri().getPath()
- }
-
- /** Wrapper around test() to be used by pyspark tests. */
- protected def pytest(desc: String)(testFn: => Unit): Unit = {
- test(desc) {
- assume(cluster.isRealSpark(), "PySpark tests require a real Spark installation.")
- testFn
- }
- }
-
- /** Wrapper around test() to be used by SparkR tests. */
- protected def rtest(desc: String)(testFn: => Unit): Unit = {
- test(desc) {
- assume(!sys.props.getOrElse("skipRTests", "false").toBoolean, "Skipping R tests.")
- assume(cluster.isRealSpark(), "SparkR tests require a real Spark installation.")
- assume(cluster.hasSparkR(), "Spark under test does not support R.")
- testFn
- }
- }
-
- /** Clean up session and show info when test fails. */
- protected def withSession[S <: LivyRestClient#Session, R]
- (s: S)
- (f: (S) => R): R = {
- try {
- f(s)
- } catch {
- case NonFatal(e) =>
- try {
- val state = s.snapshot()
- info(s"Final session state: $state")
- state.appId.foreach { id => info(s"YARN diagnostics: ${getYarnLog(id)}") }
- } catch { case NonFatal(_) => }
- throw e
- } finally {
- try {
- s.stop()
- } catch {
- case NonFatal(e) => alert(s"Failed to stop session: $e")
- }
- }
- }
-
- // We need beforeAll() here because BatchIT's beforeAll() has to be executed after this.
- // Please create an issue if this breaks test logging for cluster creation.
- protected override def beforeAll() = {
- cluster = Cluster.get()
- httpClient = new AsyncHttpClient()
- livyClient = new LivyRestClient(httpClient, livyEndpoint)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/main/scala/com/cloudera/livy/test/framework/Cluster.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/com/cloudera/livy/test/framework/Cluster.scala b/integration-test/src/main/scala/com/cloudera/livy/test/framework/Cluster.scala
deleted file mode 100644
index 06fa11d..0000000
--- a/integration-test/src/main/scala/com/cloudera/livy/test/framework/Cluster.scala
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.framework
-
-import java.io._
-import java.nio.charset.StandardCharsets.UTF_8
-import java.util.Properties
-
-import scala.collection.JavaConverters._
-import scala.util.Try
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FileSystem
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.yarn.client.api.YarnClient
-
-import com.cloudera.livy.Logging
-
-/**
- * An common interface to run test on real cluster and mini cluster.
- */
-trait Cluster {
- def deploy(): Unit
- def cleanUp(): Unit
- def configDir(): File
- def isRealSpark(): Boolean
- def hasSparkR(): Boolean
-
- def runLivy(): Unit
- def stopLivy(): Unit
- def livyEndpoint: String
- def hdfsScratchDir(): Path
-
- def doAsClusterUser[T](task: => T): T
-
- lazy val hadoopConf = {
- val conf = new Configuration(false)
- configDir().listFiles().foreach { f =>
- if (f.getName().endsWith(".xml")) {
- conf.addResource(new Path(f.toURI()))
- }
- }
- conf
- }
-
- lazy val yarnConf = {
- val conf = new Configuration(false)
- conf.addResource(new Path(s"${configDir().getCanonicalPath}/yarn-site.xml"))
- conf
- }
-
- lazy val fs = doAsClusterUser {
- FileSystem.get(hadoopConf)
- }
-
- lazy val yarnClient = doAsClusterUser {
- val c = YarnClient.createYarnClient()
- c.init(yarnConf)
- c.start()
- c
- }
-}
-
-object Cluster extends Logging {
- private val CLUSTER_TYPE = "cluster.type"
-
- private lazy val config = {
- sys.props.get("cluster.spec")
- .filter { path => path.nonEmpty && path != "default" }
- .map { path =>
- val in = Option(getClass.getClassLoader.getResourceAsStream(path))
- .getOrElse(new FileInputStream(path))
- val p = new Properties()
- val reader = new InputStreamReader(in, UTF_8)
- try {
- p.load(reader)
- } finally {
- reader.close()
- }
- p.asScala.toMap
- }
- .getOrElse(Map(CLUSTER_TYPE -> "mini"))
- }
-
- private lazy val cluster = {
- var _cluster: Cluster = null
- try {
- _cluster = config.get(CLUSTER_TYPE) match {
- case Some("real") => new RealCluster(config)
- case Some("mini") => new MiniCluster(config)
- case t => throw new Exception(s"Unknown or unset cluster.type $t")
- }
- Runtime.getRuntime.addShutdownHook(new Thread {
- override def run(): Unit = {
- info("Shutting down cluster pool.")
- _cluster.cleanUp()
- }
- })
- _cluster.deploy()
- } catch {
- case e: Throwable =>
- error("Failed to initialize cluster.", e)
- Option(_cluster).foreach { c =>
- Try(c.cleanUp()).recover { case e =>
- error("Furthermore, failed to clean up cluster after failure.", e)
- }
- }
- throw e
- }
- _cluster
- }
-
- def get(): Cluster = cluster
-
- def isRunningOnTravis: Boolean = sys.env.contains("TRAVIS")
-}
-
-trait ClusterUtils {
-
- protected def saveProperties(props: Map[String, String], dest: File): Unit = {
- val jprops = new Properties()
- props.foreach { case (k, v) => jprops.put(k, v) }
-
- val tempFile = new File(dest.getAbsolutePath() + ".tmp")
- val out = new OutputStreamWriter(new FileOutputStream(tempFile), UTF_8)
- try {
- jprops.store(out, "Configuration")
- } finally {
- out.close()
- }
- tempFile.renameTo(dest)
- }
-
- protected def loadProperties(file: File): Map[String, String] = {
- val in = new InputStreamReader(new FileInputStream(file), UTF_8)
- val props = new Properties()
- try {
- props.load(in)
- } finally {
- in.close()
- }
- props.asScala.toMap
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/main/scala/com/cloudera/livy/test/framework/LivyRestClient.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/com/cloudera/livy/test/framework/LivyRestClient.scala b/integration-test/src/main/scala/com/cloudera/livy/test/framework/LivyRestClient.scala
deleted file mode 100644
index ec63193..0000000
--- a/integration-test/src/main/scala/com/cloudera/livy/test/framework/LivyRestClient.scala
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.livy.test.framework
-
-import java.util.regex.Pattern
-import javax.servlet.http.HttpServletResponse
-
-import scala.annotation.tailrec
-import scala.concurrent.duration._
-import scala.language.postfixOps
-import scala.util.{Either, Left, Right}
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.module.scala.DefaultScalaModule
-import com.ning.http.client.AsyncHttpClient
-import com.ning.http.client.Response
-import org.apache.hadoop.yarn.api.records.ApplicationId
-import org.apache.hadoop.yarn.util.ConverterUtils
-import org.scalatest.concurrent.Eventually._
-
-import com.cloudera.livy.server.batch.CreateBatchRequest
-import com.cloudera.livy.server.interactive.CreateInteractiveRequest
-import com.cloudera.livy.sessions.{Kind, SessionKindModule, SessionState}
-import com.cloudera.livy.utils.AppInfo
-
-object LivyRestClient {
- private val BATCH_TYPE = "batches"
- private val INTERACTIVE_TYPE = "sessions"
-
- // TODO Define these in production code and share them with test code.
- @JsonIgnoreProperties(ignoreUnknown = true)
- private case class StatementResult(id: Int, state: String, output: Map[String, Any])
-
- @JsonIgnoreProperties(ignoreUnknown = true)
- case class StatementError(ename: String, evalue: String, stackTrace: Seq[String])
-
- @JsonIgnoreProperties(ignoreUnknown = true)
- case class SessionSnapshot(
- id: Int,
- appId: Option[String],
- state: String,
- appInfo: AppInfo,
- log: IndexedSeq[String])
-}
-
-class LivyRestClient(val httpClient: AsyncHttpClient, val livyEndpoint: String) {
- import LivyRestClient._
-
- val mapper = new ObjectMapper()
- .registerModule(DefaultScalaModule)
- .registerModule(new SessionKindModule())
-
- class Session(val id: Int, sessionType: String) {
- val url: String = s"$livyEndpoint/$sessionType/$id"
-
- def appId(): ApplicationId = {
- ConverterUtils.toApplicationId(snapshot().appId.get)
- }
-
- def snapshot(): SessionSnapshot = {
- val r = httpClient.prepareGet(url).execute().get()
- assertStatusCode(r, HttpServletResponse.SC_OK)
-
- mapper.readValue(r.getResponseBodyAsStream, classOf[SessionSnapshot])
- }
-
- def stop(): Unit = {
- httpClient.prepareDelete(url).execute().get()
-
- eventually(timeout(30 seconds), interval(1 second)) {
- verifySessionDoesNotExist()
- }
- }
-
- def verifySessionState(state: SessionState): Unit = {
- verifySessionState(Set(state))
- }
-
- def verifySessionState(states: Set[SessionState]): Unit = {
- val t = if (Cluster.isRunningOnTravis) 5.minutes else 2.minutes
- val strStates = states.map(_.toString)
- // Travis uses very slow VM. It needs a longer timeout.
- // Keeping the original timeout to avoid slowing down local development.
- eventually(timeout(t), interval(1 second)) {
- val s = snapshot().state
- assert(strStates.contains(s), s"Session $id state $s doesn't equal one of $strStates")
- }
- }
-
- def verifySessionDoesNotExist(): Unit = {
- val r = httpClient.prepareGet(url).execute().get()
- assertStatusCode(r, HttpServletResponse.SC_NOT_FOUND)
- }
- }
-
- class BatchSession(id: Int) extends Session(id, BATCH_TYPE) {
- def verifySessionDead(): Unit = verifySessionState(SessionState.Dead())
- def verifySessionRunning(): Unit = verifySessionState(SessionState.Running())
- def verifySessionSuccess(): Unit = verifySessionState(SessionState.Success())
- }
-
- class InteractiveSession(id: Int) extends Session(id, INTERACTIVE_TYPE) {
- class Statement(code: String) {
- val stmtId = {
- val requestBody = Map("code" -> code)
- val r = httpClient.preparePost(s"$url/statements")
- .setBody(mapper.writeValueAsString(requestBody))
- .execute()
- .get()
- assertStatusCode(r, HttpServletResponse.SC_CREATED)
-
- val newStmt = mapper.readValue(r.getResponseBodyAsStream, classOf[StatementResult])
- newStmt.id
- }
-
- final def result(): Either[String, StatementError] = {
- eventually(timeout(1 minute), interval(1 second)) {
- val r = httpClient.prepareGet(s"$url/statements/$stmtId")
- .execute()
- .get()
- assertStatusCode(r, HttpServletResponse.SC_OK)
-
- val newStmt = mapper.readValue(r.getResponseBodyAsStream, classOf[StatementResult])
- assert(newStmt.state == "available", s"Statement isn't available: ${newStmt.state}")
-
- val output = newStmt.output
- output.get("status") match {
- case Some("ok") =>
- val data = output("data").asInstanceOf[Map[String, Any]]
- var rst = data.getOrElse("text/plain", "")
- val magicRst = data.getOrElse("application/vnd.livy.table.v1+json", null)
- if (magicRst != null) {
- rst = mapper.writeValueAsString(magicRst)
- }
- Left(rst.asInstanceOf[String])
- case Some("error") => Right(mapper.convertValue(output, classOf[StatementError]))
- case Some(status) =>
- throw new IllegalStateException(s"Unknown statement $stmtId status: $status")
- case None =>
- throw new IllegalStateException(s"Unknown statement $stmtId output: $newStmt")
- }
- }
- }
-
- def verifyResult(expectedRegex: String): Unit = {
- result() match {
- case Left(result) =>
- if (expectedRegex != null) {
- matchStrings(result, expectedRegex)
- }
- case Right(error) =>
- assert(false, s"Got error from statement $stmtId $code: ${error.evalue}")
- }
- }
-
- def verifyError(
- ename: String = null, evalue: String = null, stackTrace: String = null): Unit = {
- result() match {
- case Left(result) =>
- assert(false, s"Statement $stmtId `$code` expected to fail, but succeeded.")
- case Right(error) =>
- val remoteStack = Option(error.stackTrace).getOrElse(Nil).mkString("\n")
- Seq(error.ename -> ename, error.evalue -> evalue, remoteStack -> stackTrace).foreach {
- case (actual, expected) if expected != null => matchStrings(actual, expected)
- case _ =>
- }
- }
- }
-
- private def matchStrings(actual: String, expected: String): Unit = {
- val regex = Pattern.compile(expected, Pattern.DOTALL)
- assert(regex.matcher(actual).matches(), s"$actual did not match regex $expected")
- }
- }
-
- def run(code: String): Statement = { new Statement(code) }
-
- def runFatalStatement(code: String): Unit = {
- val requestBody = Map("code" -> code)
- val r = httpClient.preparePost(s"$url/statements")
- .setBody(mapper.writeValueAsString(requestBody))
- .execute()
-
- verifySessionState(SessionState.Dead())
- }
-
- def verifySessionIdle(): Unit = {
- verifySessionState(SessionState.Idle())
- }
- }
-
- def startBatch(
- file: String,
- className: Option[String],
- args: List[String],
- sparkConf: Map[String, String]): BatchSession = {
- val r = new CreateBatchRequest()
- r.file = file
- r.className = className
- r.args = args
- r.conf = Map("spark.yarn.maxAppAttempts" -> "1") ++ sparkConf
-
- val id = start(BATCH_TYPE, mapper.writeValueAsString(r))
- new BatchSession(id)
- }
-
- def startSession(
- kind: Kind,
- sparkConf: Map[String, String],
- heartbeatTimeoutInSecond: Int): InteractiveSession = {
- val r = new CreateInteractiveRequest()
- r.kind = kind
- r.conf = sparkConf
- r.heartbeatTimeoutInSecond = heartbeatTimeoutInSecond
-
- val id = start(INTERACTIVE_TYPE, mapper.writeValueAsString(r))
- new InteractiveSession(id)
- }
-
- def connectSession(id: Int): InteractiveSession = { new InteractiveSession(id) }
-
- private def start(sessionType: String, body: String): Int = {
- val r = httpClient.preparePost(s"$livyEndpoint/$sessionType")
- .setBody(body)
- .execute()
- .get()
-
- assertStatusCode(r, HttpServletResponse.SC_CREATED)
-
- val newSession = mapper.readValue(r.getResponseBodyAsStream, classOf[SessionSnapshot])
- newSession.id
- }
-
- private def assertStatusCode(r: Response, expected: Int): Unit = {
- def pretty(r: Response): String = {
- s"${r.getStatusCode} ${r.getResponseBody}"
- }
- assert(r.getStatusCode() == expected, s"HTTP status code != $expected: ${pretty(r)}")
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/main/scala/com/cloudera/livy/test/framework/MiniCluster.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/com/cloudera/livy/test/framework/MiniCluster.scala b/integration-test/src/main/scala/com/cloudera/livy/test/framework/MiniCluster.scala
deleted file mode 100644
index edb25c6..0000000
--- a/integration-test/src/main/scala/com/cloudera/livy/test/framework/MiniCluster.scala
+++ /dev/null
@@ -1,386 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.framework
-
-import java.io._
-import java.nio.charset.Charset
-import java.nio.file.{Files, Paths}
-import javax.servlet.http.HttpServletResponse
-
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import com.ning.http.client.AsyncHttpClient
-import org.apache.commons.io.FileUtils
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.hdfs.MiniDFSCluster
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.server.MiniYARNCluster
-import org.apache.spark.launcher.SparkLauncher
-import org.scalatest.concurrent.Eventually._
-
-import com.cloudera.livy.{LivyConf, Logging}
-import com.cloudera.livy.client.common.TestUtils
-import com.cloudera.livy.server.LivyServer
-
-private class MiniClusterConfig(val config: Map[String, String]) {
-
- val nmCount = getInt("yarn.nm-count", 1)
- val localDirCount = getInt("yarn.local-dir-count", 1)
- val logDirCount = getInt("yarn.log-dir-count", 1)
- val dnCount = getInt("hdfs.dn-count", 1)
-
- private def getInt(key: String, default: Int): Int = {
- config.get(key).map(_.toInt).getOrElse(default)
- }
-
-}
-
-sealed trait MiniClusterUtils extends ClusterUtils {
- private val livySparkScalaVersionEnvVarName = "LIVY_SPARK_SCALA_VERSION"
-
- protected def getSparkScalaVersion(): String = {
- sys.env.getOrElse(livySparkScalaVersionEnvVarName, {
- throw new RuntimeException(s"Please specify env var $livySparkScalaVersionEnvVarName.")
- })
- }
-
- protected def saveConfig(conf: Configuration, dest: File): Unit = {
- val redacted = new Configuration(conf)
- // This setting references a test class that is not available when using a real Spark
- // installation, so remove it from client configs.
- redacted.unset("net.topology.node.switch.mapping.impl")
-
- val out = new FileOutputStream(dest)
- try {
- redacted.writeXml(out)
- } finally {
- out.close()
- }
- }
-
-}
-
-sealed abstract class MiniClusterBase extends MiniClusterUtils with Logging {
-
- def main(args: Array[String]): Unit = {
- val klass = getClass().getSimpleName()
-
- info(s"$klass is starting up.")
-
- val Array(configPath) = args
- val config = {
- val file = new File(s"$configPath/cluster.conf")
- val props = loadProperties(file)
- new MiniClusterConfig(props)
- }
- start(config, configPath)
-
- info(s"$klass running.")
-
- while (true) synchronized {
- wait()
- }
- }
-
- protected def start(config: MiniClusterConfig, configPath: String): Unit
-
-}
-
-object MiniHdfsMain extends MiniClusterBase {
-
- override protected def start(config: MiniClusterConfig, configPath: String): Unit = {
- val hadoopConf = new Configuration()
- val hdfsCluster = new MiniDFSCluster.Builder(hadoopConf)
- .numDataNodes(config.dnCount)
- .format(true)
- .waitSafeMode(true)
- .build()
-
- hdfsCluster.waitActive()
-
- saveConfig(hadoopConf, new File(configPath + "/core-site.xml"))
- }
-
-}
-
-object MiniYarnMain extends MiniClusterBase {
-
- override protected def start(config: MiniClusterConfig, configPath: String): Unit = {
- val baseConfig = new YarnConfiguration()
- var yarnCluster = new MiniYARNCluster(getClass().getName(), config.nmCount,
- config.localDirCount, config.logDirCount)
- yarnCluster.init(baseConfig)
-
- // Install a shutdown hook for stop the service and kill all running applications.
- Runtime.getRuntime().addShutdownHook(new Thread() {
- override def run(): Unit = yarnCluster.stop()
- })
-
- yarnCluster.start()
-
- // Workaround for YARN-2642.
- val yarnConfig = yarnCluster.getConfig()
- eventually(timeout(30 seconds), interval(100 millis)) {
- assert(yarnConfig.get(YarnConfiguration.RM_ADDRESS).split(":")(1) != "0",
- "RM not up yes.")
- }
-
- info(s"RM address in configuration is ${yarnConfig.get(YarnConfiguration.RM_ADDRESS)}")
- saveConfig(yarnConfig, new File(configPath + "/yarn-site.xml"))
- }
-
-}
-
-object MiniLivyMain extends MiniClusterBase {
- var livyUrl: Option[String] = None
-
- def start(config: MiniClusterConfig, configPath: String): Unit = {
- var livyConf = Map(
- LivyConf.LIVY_SPARK_MASTER.key -> "yarn",
- LivyConf.LIVY_SPARK_DEPLOY_MODE.key -> "cluster",
- LivyConf.LIVY_SPARK_SCALA_VERSION.key -> getSparkScalaVersion(),
- LivyConf.HEARTBEAT_WATCHDOG_INTERVAL.key -> "1s",
- LivyConf.YARN_POLL_INTERVAL.key -> "500ms",
- LivyConf.RECOVERY_MODE.key -> "recovery",
- LivyConf.RECOVERY_STATE_STORE.key -> "filesystem",
- LivyConf.RECOVERY_STATE_STORE_URL.key -> s"file://$configPath/state-store")
-
- if (Cluster.isRunningOnTravis) {
- livyConf ++= Map("livy.server.yarn.app-lookup-timeout" -> "2m")
- }
-
- saveProperties(livyConf, new File(configPath + "/livy.conf"))
-
- val server = new LivyServer()
- server.start()
- server.livyConf.set(LivyConf.ENABLE_HIVE_CONTEXT, true)
- // Write a serverUrl.conf file to the conf directory with the location of the Livy
- // server. Do it atomically since it's used by MiniCluster to detect when the Livy server
- // is up and ready.
- eventually(timeout(30 seconds), interval(1 second)) {
- val serverUrlConf = Map("livy.server.server-url" -> server.serverUrl())
- saveProperties(serverUrlConf, new File(configPath + "/serverUrl.conf"))
- }
- }
-}
-
-private case class ProcessInfo(process: Process, logFile: File)
-
-/**
- * Cluster implementation that uses HDFS / YARN mini clusters running as sub-processes locally.
- * Launching Livy through this mini cluster results in three child processes:
- *
- * - A HDFS mini cluster
- * - A YARN mini cluster
- * - The Livy server
- *
- * Each service will write its client configuration to a temporary directory managed by the
- * framework, so that applications can connect to the services.
- *
- * TODO: add support for MiniKdc.
- */
-class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterUtils with Logging {
- private val tempDir = new File(s"${sys.props("java.io.tmpdir")}/livy-int-test")
- private var sparkConfDir: File = _
- private var _configDir: File = _
- private var hdfs: Option[ProcessInfo] = None
- private var yarn: Option[ProcessInfo] = None
- private var livy: Option[ProcessInfo] = None
- private var livyUrl: String = _
- private var _hdfsScrathDir: Path = _
-
- override def configDir(): File = _configDir
-
- override def hdfsScratchDir(): Path = _hdfsScrathDir
-
- override def isRealSpark(): Boolean = {
- new File(sys.env("SPARK_HOME") + File.separator + "RELEASE").isFile()
- }
-
- override def hasSparkR(): Boolean = {
- val path = Seq(sys.env("SPARK_HOME"), "R", "lib", "sparkr.zip").mkString(File.separator)
- new File(path).isFile()
- }
-
- override def doAsClusterUser[T](task: => T): T = task
-
- // Explicitly remove the "test-lib" dependency from the classpath of child processes. We
- // want tests to explicitly upload this jar when necessary, to test those code paths.
- private val childClasspath = {
- val cp = sys.props("java.class.path").split(File.pathSeparator)
- val filtered = cp.filter { path => !new File(path).getName().startsWith("livy-test-lib-") }
- assert(cp.size != filtered.size, "livy-test-lib jar not found in classpath!")
- filtered.mkString(File.pathSeparator)
- }
-
- override def deploy(): Unit = {
- if (tempDir.exists()) {
- FileUtils.deleteQuietly(tempDir)
- }
- assert(tempDir.mkdir(), "Cannot create temp test dir.")
- sparkConfDir = mkdir("spark-conf")
-
- // When running a real Spark cluster, don't set the classpath.
- val extraCp = if (!isRealSpark()) {
- val sparkScalaVersion = getSparkScalaVersion()
- val classPathFile =
- new File(s"minicluster-dependencies/scala-$sparkScalaVersion/target/classpath")
- assert(classPathFile.isFile,
- s"Cannot read MiniCluster classpath file: ${classPathFile.getCanonicalPath}")
- val sparkClassPath =
- FileUtils.readFileToString(classPathFile, Charset.defaultCharset())
-
- val dummyJar = Files.createTempFile(Paths.get(tempDir.toURI), "dummy", "jar").toFile
- Map(
- SparkLauncher.DRIVER_EXTRA_CLASSPATH -> sparkClassPath,
- SparkLauncher.EXECUTOR_EXTRA_CLASSPATH -> sparkClassPath,
- // Used for Spark 2.0. Spark 2.0 will upload specified jars to distributed cache in yarn
- // mode, if not specified it will check jars folder. Here since jars folder is not
- // existed, so it will throw exception.
- "spark.yarn.jars" -> dummyJar.getAbsolutePath)
- } else {
- Map()
- }
-
- val sparkConf = extraCp ++ Map(
- "spark.executor.instances" -> "1",
- "spark.scheduler.minRegisteredResourcesRatio" -> "0.0",
- "spark.ui.enabled" -> "false",
- SparkLauncher.DRIVER_MEMORY -> "512m",
- SparkLauncher.EXECUTOR_MEMORY -> "512m",
- SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS -> "-Dtest.appender=console",
- SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS -> "-Dtest.appender=console"
- )
- saveProperties(sparkConf, new File(sparkConfDir, "spark-defaults.conf"))
-
- _configDir = mkdir("hadoop-conf")
- saveProperties(config, new File(configDir, "cluster.conf"))
- hdfs = Some(start(MiniHdfsMain.getClass, new File(configDir, "core-site.xml")))
- yarn = Some(start(MiniYarnMain.getClass, new File(configDir, "yarn-site.xml")))
- runLivy()
-
- _hdfsScrathDir = fs.makeQualified(new Path("/"))
- }
-
- override def cleanUp(): Unit = {
- Seq(hdfs, yarn, livy).flatten.foreach(stop)
- hdfs = None
- yarn = None
- livy = None
- }
-
- def runLivy(): Unit = {
- assert(!livy.isDefined)
- val confFile = new File(configDir, "serverUrl.conf")
- val jacocoArgs = Option(TestUtils.getJacocoArgs())
- .map { args =>
- Seq(args, s"-Djacoco.args=$args")
- }.getOrElse(Nil)
- val localLivy = start(MiniLivyMain.getClass, confFile, extraJavaArgs = jacocoArgs)
-
- val props = loadProperties(confFile)
- livyUrl = props("livy.server.server-url")
-
- // Wait until Livy server responds.
- val httpClient = new AsyncHttpClient()
- eventually(timeout(30 seconds), interval(1 second)) {
- val res = httpClient.prepareGet(livyUrl + "/metrics").execute().get()
- assert(res.getStatusCode() == HttpServletResponse.SC_OK)
- }
-
- livy = Some(localLivy)
- }
-
- def stopLivy(): Unit = {
- assert(livy.isDefined)
- livy.foreach(stop)
- livyUrl = null
- livy = None
- }
-
- def livyEndpoint: String = livyUrl
-
- private def mkdir(name: String, parent: File = tempDir): File = {
- val dir = new File(parent, name)
- if (!dir.exists()) {
- assert(dir.mkdir(), s"Failed to create directory $name.")
- }
- dir
- }
-
- private def start(
- klass: Class[_],
- configFile: File,
- extraJavaArgs: Seq[String] = Nil): ProcessInfo = {
- val simpleName = klass.getSimpleName().stripSuffix("$")
- val procDir = mkdir(simpleName)
- val procTmp = mkdir("tmp", parent = procDir)
-
- // Before starting anything, clean up previous running sessions.
- sys.process.Process(s"pkill -f $simpleName") !
-
- val java = sys.props("java.home") + "/bin/java"
- val cmd =
- Seq(
- sys.props("java.home") + "/bin/java",
- "-Dtest.appender=console",
- "-Djava.io.tmpdir=" + procTmp.getAbsolutePath(),
- "-cp", childClasspath + File.pathSeparator + configDir.getAbsolutePath(),
- "-XX:MaxPermSize=256m") ++
- extraJavaArgs ++
- Seq(
- klass.getName().stripSuffix("$"),
- configDir.getAbsolutePath())
-
- val logFile = new File(procDir, "output.log")
- val pb = new ProcessBuilder(cmd.toArray: _*)
- .directory(procDir)
- .redirectErrorStream(true)
- .redirectOutput(ProcessBuilder.Redirect.appendTo(logFile))
-
- pb.environment().put("LIVY_CONF_DIR", configDir.getAbsolutePath())
- pb.environment().put("HADOOP_CONF_DIR", configDir.getAbsolutePath())
- pb.environment().put("SPARK_CONF_DIR", sparkConfDir.getAbsolutePath())
- pb.environment().put("SPARK_LOCAL_IP", "127.0.0.1")
-
- val child = pb.start()
-
- // Wait for the config file to show up before returning, so that dependent services
- // can see the configuration. Exit early if process dies.
- eventually(timeout(30 seconds), interval(100 millis)) {
- assert(configFile.isFile(), s"$simpleName hasn't started yet.")
-
- try {
- val exitCode = child.exitValue()
- throw new IOException(s"Child process exited unexpectedly (exit code $exitCode)")
- } catch {
- case _: IllegalThreadStateException => // Try again.
- }
- }
-
- ProcessInfo(child, logFile)
- }
-
- private def stop(svc: ProcessInfo): Unit = {
- svc.process.destroy()
- svc.process.waitFor()
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/main/scala/com/cloudera/livy/test/framework/RealCluster.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/com/cloudera/livy/test/framework/RealCluster.scala b/integration-test/src/main/scala/com/cloudera/livy/test/framework/RealCluster.scala
deleted file mode 100644
index 9c79829..0000000
--- a/integration-test/src/main/scala/com/cloudera/livy/test/framework/RealCluster.scala
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.test.framework
-
-import java.io.{File, IOException}
-import java.nio.file.Files
-import java.security.PrivilegedExceptionAction
-import javax.servlet.http.HttpServletResponse._
-
-import scala.collection.JavaConverters._
-import scala.concurrent.duration._
-import scala.language.postfixOps
-import scala.sys.process._
-import scala.util.Random
-
-import com.decodified.scalassh._
-import com.ning.http.client.AsyncHttpClient
-import org.apache.hadoop.fs.{FileSystem, Path}
-import org.apache.hadoop.fs.permission.FsPermission
-import org.apache.hadoop.security.UserGroupInformation
-import org.scalatest.concurrent.Eventually._
-
-import com.cloudera.livy.{LivyConf, Logging}
-
-private class RealClusterConfig(config: Map[String, String]) {
- val ip = config("ip")
-
- val sshLogin = config("ssh.login")
- val sshPubKey = config("ssh.pubkey")
- val livyPort = config.getOrElse(LivyConf.SERVER_PORT.key, "8998").toInt
- val livyClasspath = config.getOrElse("livy.classpath", "")
-
- val deployLivy = config.getOrElse("deploy-livy", "true").toBoolean
- val noDeployLivyHome = config.get("livy-home")
-
- val sparkHome = config("env.spark_home")
- val sparkConf = config.getOrElse("env.spark_conf", "/etc/spark/conf")
- val hadoopConf = config.getOrElse("env.hadoop_conf", "/etc/hadoop/conf")
-
- val javaHome = config.getOrElse("env.java_home", "/usr/java/default")
-}
-
-class RealCluster(_config: Map[String, String])
- extends Cluster with ClusterUtils with Logging {
-
- private val config = new RealClusterConfig(_config)
-
- private var livyIsRunning = false
- private var livyHomePath: String = _
- private var livyEpoch = 0
-
- private var _configDir: File = _
-
- private var hdfsScratch: Path = _
-
- private var sparkConfDir: String = _
- private var tempDirPath: String = _
- private var _hasSparkR: Boolean = _
-
- override def isRealSpark(): Boolean = true
-
- override def hasSparkR(): Boolean = _hasSparkR
-
- override def configDir(): File = _configDir
-
- override def hdfsScratchDir(): Path = hdfsScratch
-
- override def doAsClusterUser[T](task: => T): T = {
- val user = UserGroupInformation.createRemoteUser(config.sshLogin)
- user.doAs(new PrivilegedExceptionAction[T] {
- override def run(): T = task
- })
- }
-
- private def sshClient[T](body: SshClient => SSH.Result[T]): T = {
- val sshLogin = PublicKeyLogin(
- config.sshLogin, None, config.sshPubKey :: Nil)
- val hostConfig = HostConfig(login = sshLogin, hostKeyVerifier = HostKeyVerifiers.DontVerify)
- SSH(config.ip, hostConfig)(body) match {
- case Left(err) => throw new IOException(err)
- case Right(result) => result
- }
- }
-
- private def exec(cmd: String): CommandResult = {
- info(s"Running command: $cmd")
- val result = sshClient(_.exec(cmd))
- result.exitCode match {
- case Some(ec) if ec > 0 =>
- throw new IOException(s"Command '$cmd' failed: $ec\n" +
- s"stdout: ${result.stdOutAsString()}\n" +
- s"stderr: ${result.stdErrAsString()}\n")
- case _ =>
- }
- result
- }
-
- private def upload(local: String, remote: String): Unit = {
- info(s"Uploading local path $local")
- sshClient(_.upload(local, remote))
- }
-
- private def download(remote: String, local: String): Unit = {
- info(s"Downloading remote path $remote")
- sshClient(_.download(remote, local))
- }
-
- override def deploy(): Unit = {
- // Make sure Livy is not running.
- stopLivy()
-
- // Check whether SparkR is supported in YARN (need the sparkr.zip archive).\
- _hasSparkR = try {
- exec(s"test -f ${config.sparkHome}/R/lib/sparkr.zip")
- true
- } catch {
- case e: IOException => false
- }
-
- // Copy the remove Hadoop configuration to a local temp dir so that tests can use it to
- // talk to HDFS and YARN.
- val localTemp = new File(sys.props("java.io.tmpdir") + File.separator + "hadoop-conf")
- download(config.hadoopConf, localTemp.getAbsolutePath())
- _configDir = localTemp
-
- // Create a temp directory where test files will be written.
- tempDirPath = s"/tmp/livy-it-${Random.alphanumeric.take(16).mkString}"
- exec(s"mkdir -p $tempDirPath")
-
- // Also create an HDFS scratch directory for tests.
- doAsClusterUser {
- hdfsScratch = fs.makeQualified(
- new Path(s"/tmp/livy-it-${Random.alphanumeric.take(16).mkString}"))
- fs.mkdirs(hdfsScratch)
- fs.setPermission(hdfsScratch, new FsPermission("777"))
- }
-
- // Create a copy of the Spark configuration, and make sure the master is "yarn-cluster".
- sparkConfDir = s"$tempDirPath/spark-conf"
- val sparkProps = s"$sparkConfDir/spark-defaults.conf"
- Seq(
- s"cp -Lr ${config.sparkConf} $sparkConfDir",
- s"touch $sparkProps",
- s"sed -i.old '/spark.master.*/d' $sparkProps",
- s"sed -i.old '/spark.submit.deployMode.*/d' $sparkProps",
- s"echo 'spark.master=yarn-cluster' >> $sparkProps"
- ).foreach(exec)
-
- if (config.deployLivy) {
- try {
- info(s"Deploying Livy to ${config.ip}...")
- val version = sys.props("project.version")
- val assemblyZip = new File(s"../assembly/target/livy-server-$version.zip")
- assert(assemblyZip.isFile,
- s"Can't find livy assembly zip at ${assemblyZip.getCanonicalPath}")
- val assemblyName = assemblyZip.getName()
-
- // SSH to the node to unzip and install Livy.
- upload(assemblyZip.getCanonicalPath, s"$tempDirPath/$assemblyName")
- exec(s"unzip -o $tempDirPath/$assemblyName -d $tempDirPath")
- livyHomePath = s"$tempDirPath/livy-server-$version"
- info(s"Deployed Livy to ${config.ip} at $livyHomePath.")
- } catch {
- case e: Exception =>
- error(s"Failed to deploy Livy to ${config.ip}.", e)
- cleanUp()
- throw e
- }
- } else {
- livyHomePath = config.noDeployLivyHome.get
- info("Skipping deployment.")
- }
-
- runLivy()
- }
-
- override def cleanUp(): Unit = {
- stopLivy()
- if (tempDirPath != null) {
- exec(s"rm -rf $tempDirPath")
- }
- if (hdfsScratch != null) {
- doAsClusterUser {
- // Cannot use the shared `fs` since this runs in a shutdown hook, and that instance
- // may have been closed already.
- val fs = FileSystem.newInstance(hadoopConf)
- try {
- fs.delete(hdfsScratch, true)
- } finally {
- fs.close()
- }
- }
- }
- }
-
- override def runLivy(): Unit = synchronized {
- assert(!livyIsRunning, "Livy is running already.")
-
- val livyConf = Map(
- "livy.server.port" -> config.livyPort.toString,
- // "livy.server.recovery.mode=local",
- "livy.environment" -> "development",
- LivyConf.LIVY_SPARK_MASTER.key -> "yarn",
- LivyConf.LIVY_SPARK_DEPLOY_MODE.key -> "cluster")
- val livyConfFile = File.createTempFile("livy.", ".properties")
- saveProperties(livyConf, livyConfFile)
- upload(livyConfFile.getAbsolutePath(), s"$livyHomePath/conf/livy.conf")
-
- val env = Map(
- "JAVA_HOME" -> config.javaHome,
- "HADOOP_CONF_DIR" -> config.hadoopConf,
- "SPARK_CONF_DIR" -> sparkConfDir,
- "SPARK_HOME" -> config.sparkHome,
- "CLASSPATH" -> config.livyClasspath,
- "LIVY_PID_DIR" -> s"$tempDirPath/pid",
- "LIVY_LOG_DIR" -> s"$tempDirPath/logs",
- "LIVY_MAX_LOG_FILES" -> "16",
- "LIVY_IDENT_STRING" -> "it"
- )
- val livyEnvFile = File.createTempFile("livy-env.", ".sh")
- saveProperties(env, livyEnvFile)
- upload(livyEnvFile.getAbsolutePath(), s"$livyHomePath/conf/livy-env.sh")
-
- info(s"Starting Livy @ port ${config.livyPort}...")
- exec(s"env -i $livyHomePath/bin/livy-server start")
- livyIsRunning = true
-
- val httpClient = new AsyncHttpClient()
- eventually(timeout(1 minute), interval(1 second)) {
- assert(httpClient.prepareGet(livyEndpoint).execute().get().getStatusCode == SC_OK)
- }
- info(s"Started Livy.")
- }
-
- override def stopLivy(): Unit = synchronized {
- info("Stopping Livy Server")
- try {
- exec(s"$livyHomePath/bin/livy-server stop")
- } catch {
- case e: Exception =>
- if (livyIsRunning) {
- throw e
- }
- }
-
- if (livyIsRunning) {
- // Wait a tiny bit so that the process finishes closing its output files.
- Thread.sleep(2)
-
- livyEpoch += 1
- val logName = "livy-it-server.out"
- val localName = s"livy-it-server-$livyEpoch.out"
- val logPath = s"$tempDirPath/logs/$logName"
- val localLog = sys.props("java.io.tmpdir") + File.separator + localName
- download(logPath, localLog)
- info(s"Log for epoch $livyEpoch available at $localLog")
- }
- }
-
- override def livyEndpoint: String = s"http://${config.ip}:${config.livyPort}"
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/main/scala/org/apache/livy/test/framework/BaseIntegrationTestSuite.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/BaseIntegrationTestSuite.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/BaseIntegrationTestSuite.scala
new file mode 100644
index 0000000..d770e7e
--- /dev/null
+++ b/integration-test/src/main/scala/org/apache/livy/test/framework/BaseIntegrationTestSuite.scala
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.framework
+
+import java.io.File
+import java.util.UUID
+
+import scala.concurrent._
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.util.control.NonFatal
+
+import com.ning.http.client.AsyncHttpClient
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.util.ConverterUtils
+import org.scalatest._
+
+abstract class BaseIntegrationTestSuite extends FunSuite with Matchers with BeforeAndAfterAll {
+ import scala.concurrent.ExecutionContext.Implicits.global
+
+ var cluster: Cluster = _
+ var httpClient: AsyncHttpClient = _
+ var livyClient: LivyRestClient = _
+
+ protected def livyEndpoint: String = cluster.livyEndpoint
+
+ protected val testLib = sys.props("java.class.path")
+ .split(File.pathSeparator)
+ .find(new File(_).getName().startsWith("livy-test-lib-"))
+ .getOrElse(throw new Exception(s"Cannot find test lib in ${sys.props("java.class.path")}"))
+
+ protected def getYarnLog(appId: String): String = {
+ require(appId != null, "appId shouldn't be null")
+
+ val appReport = cluster.yarnClient.getApplicationReport(ConverterUtils.toApplicationId(appId))
+ assert(appReport != null, "appReport shouldn't be null")
+
+ appReport.getDiagnostics()
+ }
+
+ protected def restartLivy(): Unit = {
+ val f = future {
+ cluster.stopLivy()
+ cluster.runLivy()
+ }
+ Await.result(f, 3 minutes)
+ }
+
+ /** Uploads a file to HDFS and returns just its path. */
+ protected def uploadToHdfs(file: File): String = {
+ val hdfsPath = new Path(cluster.hdfsScratchDir(),
+ UUID.randomUUID().toString() + "-" + file.getName())
+ cluster.fs.copyFromLocalFile(new Path(file.toURI()), hdfsPath)
+ hdfsPath.toUri().getPath()
+ }
+
+ /** Wrapper around test() to be used by pyspark tests. */
+ protected def pytest(desc: String)(testFn: => Unit): Unit = {
+ test(desc) {
+ assume(cluster.isRealSpark(), "PySpark tests require a real Spark installation.")
+ testFn
+ }
+ }
+
+ /** Wrapper around test() to be used by SparkR tests. */
+ protected def rtest(desc: String)(testFn: => Unit): Unit = {
+ test(desc) {
+ assume(!sys.props.getOrElse("skipRTests", "false").toBoolean, "Skipping R tests.")
+ assume(cluster.isRealSpark(), "SparkR tests require a real Spark installation.")
+ assume(cluster.hasSparkR(), "Spark under test does not support R.")
+ testFn
+ }
+ }
+
+ /** Clean up session and show info when test fails. */
+ protected def withSession[S <: LivyRestClient#Session, R]
+ (s: S)
+ (f: (S) => R): R = {
+ try {
+ f(s)
+ } catch {
+ case NonFatal(e) =>
+ try {
+ val state = s.snapshot()
+ info(s"Final session state: $state")
+ state.appId.foreach { id => info(s"YARN diagnostics: ${getYarnLog(id)}") }
+ } catch { case NonFatal(_) => }
+ throw e
+ } finally {
+ try {
+ s.stop()
+ } catch {
+ case NonFatal(e) => alert(s"Failed to stop session: $e")
+ }
+ }
+ }
+
+ // We need beforeAll() here because BatchIT's beforeAll() has to be executed after this.
+ // Please create an issue if this breaks test logging for cluster creation.
+ protected override def beforeAll() = {
+ cluster = Cluster.get()
+ httpClient = new AsyncHttpClient()
+ livyClient = new LivyRestClient(httpClient, livyEndpoint)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/main/scala/org/apache/livy/test/framework/Cluster.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/Cluster.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/Cluster.scala
new file mode 100644
index 0000000..e1b6844
--- /dev/null
+++ b/integration-test/src/main/scala/org/apache/livy/test/framework/Cluster.scala
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.framework
+
+import java.io._
+import java.nio.charset.StandardCharsets.UTF_8
+import java.util.Properties
+
+import scala.collection.JavaConverters._
+import scala.util.Try
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.client.api.YarnClient
+
+import org.apache.livy.Logging
+
+/**
+ * An common interface to run test on real cluster and mini cluster.
+ */
+trait Cluster {
+ def deploy(): Unit
+ def cleanUp(): Unit
+ def configDir(): File
+ def isRealSpark(): Boolean
+ def hasSparkR(): Boolean
+
+ def runLivy(): Unit
+ def stopLivy(): Unit
+ def livyEndpoint: String
+ def hdfsScratchDir(): Path
+
+ def doAsClusterUser[T](task: => T): T
+
+ lazy val hadoopConf = {
+ val conf = new Configuration(false)
+ configDir().listFiles().foreach { f =>
+ if (f.getName().endsWith(".xml")) {
+ conf.addResource(new Path(f.toURI()))
+ }
+ }
+ conf
+ }
+
+ lazy val yarnConf = {
+ val conf = new Configuration(false)
+ conf.addResource(new Path(s"${configDir().getCanonicalPath}/yarn-site.xml"))
+ conf
+ }
+
+ lazy val fs = doAsClusterUser {
+ FileSystem.get(hadoopConf)
+ }
+
+ lazy val yarnClient = doAsClusterUser {
+ val c = YarnClient.createYarnClient()
+ c.init(yarnConf)
+ c.start()
+ c
+ }
+}
+
+object Cluster extends Logging {
+ private val CLUSTER_TYPE = "cluster.type"
+
+ private lazy val config = {
+ sys.props.get("cluster.spec")
+ .filter { path => path.nonEmpty && path != "default" }
+ .map { path =>
+ val in = Option(getClass.getClassLoader.getResourceAsStream(path))
+ .getOrElse(new FileInputStream(path))
+ val p = new Properties()
+ val reader = new InputStreamReader(in, UTF_8)
+ try {
+ p.load(reader)
+ } finally {
+ reader.close()
+ }
+ p.asScala.toMap
+ }
+ .getOrElse(Map(CLUSTER_TYPE -> "mini"))
+ }
+
+ private lazy val cluster = {
+ var _cluster: Cluster = null
+ try {
+ _cluster = config.get(CLUSTER_TYPE) match {
+ case Some("real") => new RealCluster(config)
+ case Some("mini") => new MiniCluster(config)
+ case t => throw new Exception(s"Unknown or unset cluster.type $t")
+ }
+ Runtime.getRuntime.addShutdownHook(new Thread {
+ override def run(): Unit = {
+ info("Shutting down cluster pool.")
+ _cluster.cleanUp()
+ }
+ })
+ _cluster.deploy()
+ } catch {
+ case e: Throwable =>
+ error("Failed to initialize cluster.", e)
+ Option(_cluster).foreach { c =>
+ Try(c.cleanUp()).recover { case e =>
+ error("Furthermore, failed to clean up cluster after failure.", e)
+ }
+ }
+ throw e
+ }
+ _cluster
+ }
+
+ def get(): Cluster = cluster
+
+ def isRunningOnTravis: Boolean = sys.env.contains("TRAVIS")
+}
+
+trait ClusterUtils {
+
+ protected def saveProperties(props: Map[String, String], dest: File): Unit = {
+ val jprops = new Properties()
+ props.foreach { case (k, v) => jprops.put(k, v) }
+
+ val tempFile = new File(dest.getAbsolutePath() + ".tmp")
+ val out = new OutputStreamWriter(new FileOutputStream(tempFile), UTF_8)
+ try {
+ jprops.store(out, "Configuration")
+ } finally {
+ out.close()
+ }
+ tempFile.renameTo(dest)
+ }
+
+ protected def loadProperties(file: File): Map[String, String] = {
+ val in = new InputStreamReader(new FileInputStream(file), UTF_8)
+ val props = new Properties()
+ try {
+ props.load(in)
+ } finally {
+ in.close()
+ }
+ props.asScala.toMap
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/main/scala/org/apache/livy/test/framework/LivyRestClient.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/LivyRestClient.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/LivyRestClient.scala
new file mode 100644
index 0000000..6d319c7
--- /dev/null
+++ b/integration-test/src/main/scala/org/apache/livy/test/framework/LivyRestClient.scala
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.livy.test.framework
+
+import java.util.regex.Pattern
+import javax.servlet.http.HttpServletResponse
+
+import scala.annotation.tailrec
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.util.{Either, Left, Right}
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+import com.ning.http.client.AsyncHttpClient
+import com.ning.http.client.Response
+import org.apache.hadoop.yarn.api.records.ApplicationId
+import org.apache.hadoop.yarn.util.ConverterUtils
+import org.scalatest.concurrent.Eventually._
+
+import org.apache.livy.server.batch.CreateBatchRequest
+import org.apache.livy.server.interactive.CreateInteractiveRequest
+import org.apache.livy.sessions.{Kind, SessionKindModule, SessionState}
+import org.apache.livy.utils.AppInfo
+
+object LivyRestClient {
+ private val BATCH_TYPE = "batches"
+ private val INTERACTIVE_TYPE = "sessions"
+
+ // TODO Define these in production code and share them with test code.
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ private case class StatementResult(id: Int, state: String, output: Map[String, Any])
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ case class StatementError(ename: String, evalue: String, stackTrace: Seq[String])
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ case class SessionSnapshot(
+ id: Int,
+ appId: Option[String],
+ state: String,
+ appInfo: AppInfo,
+ log: IndexedSeq[String])
+}
+
+class LivyRestClient(val httpClient: AsyncHttpClient, val livyEndpoint: String) {
+ import LivyRestClient._
+
+ val mapper = new ObjectMapper()
+ .registerModule(DefaultScalaModule)
+ .registerModule(new SessionKindModule())
+
+ class Session(val id: Int, sessionType: String) {
+ val url: String = s"$livyEndpoint/$sessionType/$id"
+
+ def appId(): ApplicationId = {
+ ConverterUtils.toApplicationId(snapshot().appId.get)
+ }
+
+ def snapshot(): SessionSnapshot = {
+ val r = httpClient.prepareGet(url).execute().get()
+ assertStatusCode(r, HttpServletResponse.SC_OK)
+
+ mapper.readValue(r.getResponseBodyAsStream, classOf[SessionSnapshot])
+ }
+
+ def stop(): Unit = {
+ httpClient.prepareDelete(url).execute().get()
+
+ eventually(timeout(30 seconds), interval(1 second)) {
+ verifySessionDoesNotExist()
+ }
+ }
+
+ def verifySessionState(state: SessionState): Unit = {
+ verifySessionState(Set(state))
+ }
+
+ def verifySessionState(states: Set[SessionState]): Unit = {
+ val t = if (Cluster.isRunningOnTravis) 5.minutes else 2.minutes
+ val strStates = states.map(_.toString)
+ // Travis uses very slow VM. It needs a longer timeout.
+ // Keeping the original timeout to avoid slowing down local development.
+ eventually(timeout(t), interval(1 second)) {
+ val s = snapshot().state
+ assert(strStates.contains(s), s"Session $id state $s doesn't equal one of $strStates")
+ }
+ }
+
+ def verifySessionDoesNotExist(): Unit = {
+ val r = httpClient.prepareGet(url).execute().get()
+ assertStatusCode(r, HttpServletResponse.SC_NOT_FOUND)
+ }
+ }
+
+ class BatchSession(id: Int) extends Session(id, BATCH_TYPE) {
+ def verifySessionDead(): Unit = verifySessionState(SessionState.Dead())
+ def verifySessionRunning(): Unit = verifySessionState(SessionState.Running())
+ def verifySessionSuccess(): Unit = verifySessionState(SessionState.Success())
+ }
+
+ class InteractiveSession(id: Int) extends Session(id, INTERACTIVE_TYPE) {
+ class Statement(code: String) {
+ val stmtId = {
+ val requestBody = Map("code" -> code)
+ val r = httpClient.preparePost(s"$url/statements")
+ .setBody(mapper.writeValueAsString(requestBody))
+ .execute()
+ .get()
+ assertStatusCode(r, HttpServletResponse.SC_CREATED)
+
+ val newStmt = mapper.readValue(r.getResponseBodyAsStream, classOf[StatementResult])
+ newStmt.id
+ }
+
+ final def result(): Either[String, StatementError] = {
+ eventually(timeout(1 minute), interval(1 second)) {
+ val r = httpClient.prepareGet(s"$url/statements/$stmtId")
+ .execute()
+ .get()
+ assertStatusCode(r, HttpServletResponse.SC_OK)
+
+ val newStmt = mapper.readValue(r.getResponseBodyAsStream, classOf[StatementResult])
+ assert(newStmt.state == "available", s"Statement isn't available: ${newStmt.state}")
+
+ val output = newStmt.output
+ output.get("status") match {
+ case Some("ok") =>
+ val data = output("data").asInstanceOf[Map[String, Any]]
+ var rst = data.getOrElse("text/plain", "")
+ val magicRst = data.getOrElse("application/vnd.livy.table.v1+json", null)
+ if (magicRst != null) {
+ rst = mapper.writeValueAsString(magicRst)
+ }
+ Left(rst.asInstanceOf[String])
+ case Some("error") => Right(mapper.convertValue(output, classOf[StatementError]))
+ case Some(status) =>
+ throw new IllegalStateException(s"Unknown statement $stmtId status: $status")
+ case None =>
+ throw new IllegalStateException(s"Unknown statement $stmtId output: $newStmt")
+ }
+ }
+ }
+
+ def verifyResult(expectedRegex: String): Unit = {
+ result() match {
+ case Left(result) =>
+ if (expectedRegex != null) {
+ matchStrings(result, expectedRegex)
+ }
+ case Right(error) =>
+ assert(false, s"Got error from statement $stmtId $code: ${error.evalue}")
+ }
+ }
+
+ def verifyError(
+ ename: String = null, evalue: String = null, stackTrace: String = null): Unit = {
+ result() match {
+ case Left(result) =>
+ assert(false, s"Statement $stmtId `$code` expected to fail, but succeeded.")
+ case Right(error) =>
+ val remoteStack = Option(error.stackTrace).getOrElse(Nil).mkString("\n")
+ Seq(error.ename -> ename, error.evalue -> evalue, remoteStack -> stackTrace).foreach {
+ case (actual, expected) if expected != null => matchStrings(actual, expected)
+ case _ =>
+ }
+ }
+ }
+
+ private def matchStrings(actual: String, expected: String): Unit = {
+ val regex = Pattern.compile(expected, Pattern.DOTALL)
+ assert(regex.matcher(actual).matches(), s"$actual did not match regex $expected")
+ }
+ }
+
+ def run(code: String): Statement = { new Statement(code) }
+
+ def runFatalStatement(code: String): Unit = {
+ val requestBody = Map("code" -> code)
+ val r = httpClient.preparePost(s"$url/statements")
+ .setBody(mapper.writeValueAsString(requestBody))
+ .execute()
+
+ verifySessionState(SessionState.Dead())
+ }
+
+ def verifySessionIdle(): Unit = {
+ verifySessionState(SessionState.Idle())
+ }
+ }
+
+ def startBatch(
+ file: String,
+ className: Option[String],
+ args: List[String],
+ sparkConf: Map[String, String]): BatchSession = {
+ val r = new CreateBatchRequest()
+ r.file = file
+ r.className = className
+ r.args = args
+ r.conf = Map("spark.yarn.maxAppAttempts" -> "1") ++ sparkConf
+
+ val id = start(BATCH_TYPE, mapper.writeValueAsString(r))
+ new BatchSession(id)
+ }
+
+ def startSession(
+ kind: Kind,
+ sparkConf: Map[String, String],
+ heartbeatTimeoutInSecond: Int): InteractiveSession = {
+ val r = new CreateInteractiveRequest()
+ r.kind = kind
+ r.conf = sparkConf
+ r.heartbeatTimeoutInSecond = heartbeatTimeoutInSecond
+
+ val id = start(INTERACTIVE_TYPE, mapper.writeValueAsString(r))
+ new InteractiveSession(id)
+ }
+
+ def connectSession(id: Int): InteractiveSession = { new InteractiveSession(id) }
+
+ private def start(sessionType: String, body: String): Int = {
+ val r = httpClient.preparePost(s"$livyEndpoint/$sessionType")
+ .setBody(body)
+ .execute()
+ .get()
+
+ assertStatusCode(r, HttpServletResponse.SC_CREATED)
+
+ val newSession = mapper.readValue(r.getResponseBodyAsStream, classOf[SessionSnapshot])
+ newSession.id
+ }
+
+ private def assertStatusCode(r: Response, expected: Int): Unit = {
+ def pretty(r: Response): String = {
+ s"${r.getStatusCode} ${r.getResponseBody}"
+ }
+ assert(r.getStatusCode() == expected, s"HTTP status code != $expected: ${pretty(r)}")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
new file mode 100644
index 0000000..005a3e9
--- /dev/null
+++ b/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
@@ -0,0 +1,386 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.test.framework
+
+import java.io._
+import java.nio.charset.Charset
+import java.nio.file.{Files, Paths}
+import javax.servlet.http.HttpServletResponse
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import com.ning.http.client.AsyncHttpClient
+import org.apache.commons.io.FileUtils
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hdfs.MiniDFSCluster
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.server.MiniYARNCluster
+import org.apache.spark.launcher.SparkLauncher
+import org.scalatest.concurrent.Eventually._
+
+import org.apache.livy.{LivyConf, Logging}
+import org.apache.livy.client.common.TestUtils
+import org.apache.livy.server.LivyServer
+
+private class MiniClusterConfig(val config: Map[String, String]) {
+
+ val nmCount = getInt("yarn.nm-count", 1)
+ val localDirCount = getInt("yarn.local-dir-count", 1)
+ val logDirCount = getInt("yarn.log-dir-count", 1)
+ val dnCount = getInt("hdfs.dn-count", 1)
+
+ private def getInt(key: String, default: Int): Int = {
+ config.get(key).map(_.toInt).getOrElse(default)
+ }
+
+}
+
+sealed trait MiniClusterUtils extends ClusterUtils {
+ private val livySparkScalaVersionEnvVarName = "LIVY_SPARK_SCALA_VERSION"
+
+ protected def getSparkScalaVersion(): String = {
+ sys.env.getOrElse(livySparkScalaVersionEnvVarName, {
+ throw new RuntimeException(s"Please specify env var $livySparkScalaVersionEnvVarName.")
+ })
+ }
+
+ protected def saveConfig(conf: Configuration, dest: File): Unit = {
+ val redacted = new Configuration(conf)
+ // This setting references a test class that is not available when using a real Spark
+ // installation, so remove it from client configs.
+ redacted.unset("net.topology.node.switch.mapping.impl")
+
+ val out = new FileOutputStream(dest)
+ try {
+ redacted.writeXml(out)
+ } finally {
+ out.close()
+ }
+ }
+
+}
+
+sealed abstract class MiniClusterBase extends MiniClusterUtils with Logging {
+
+ def main(args: Array[String]): Unit = {
+ val klass = getClass().getSimpleName()
+
+ info(s"$klass is starting up.")
+
+ val Array(configPath) = args
+ val config = {
+ val file = new File(s"$configPath/cluster.conf")
+ val props = loadProperties(file)
+ new MiniClusterConfig(props)
+ }
+ start(config, configPath)
+
+ info(s"$klass running.")
+
+ while (true) synchronized {
+ wait()
+ }
+ }
+
+ protected def start(config: MiniClusterConfig, configPath: String): Unit
+
+}
+
+object MiniHdfsMain extends MiniClusterBase {
+
+ override protected def start(config: MiniClusterConfig, configPath: String): Unit = {
+ val hadoopConf = new Configuration()
+ val hdfsCluster = new MiniDFSCluster.Builder(hadoopConf)
+ .numDataNodes(config.dnCount)
+ .format(true)
+ .waitSafeMode(true)
+ .build()
+
+ hdfsCluster.waitActive()
+
+ saveConfig(hadoopConf, new File(configPath + "/core-site.xml"))
+ }
+
+}
+
+object MiniYarnMain extends MiniClusterBase {
+
+ override protected def start(config: MiniClusterConfig, configPath: String): Unit = {
+ val baseConfig = new YarnConfiguration()
+ var yarnCluster = new MiniYARNCluster(getClass().getName(), config.nmCount,
+ config.localDirCount, config.logDirCount)
+ yarnCluster.init(baseConfig)
+
+ // Install a shutdown hook for stop the service and kill all running applications.
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ override def run(): Unit = yarnCluster.stop()
+ })
+
+ yarnCluster.start()
+
+ // Workaround for YARN-2642.
+ val yarnConfig = yarnCluster.getConfig()
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ assert(yarnConfig.get(YarnConfiguration.RM_ADDRESS).split(":")(1) != "0",
+ "RM not up yes.")
+ }
+
+ info(s"RM address in configuration is ${yarnConfig.get(YarnConfiguration.RM_ADDRESS)}")
+ saveConfig(yarnConfig, new File(configPath + "/yarn-site.xml"))
+ }
+
+}
+
+object MiniLivyMain extends MiniClusterBase {
+ var livyUrl: Option[String] = None
+
+ def start(config: MiniClusterConfig, configPath: String): Unit = {
+ var livyConf = Map(
+ LivyConf.LIVY_SPARK_MASTER.key -> "yarn",
+ LivyConf.LIVY_SPARK_DEPLOY_MODE.key -> "cluster",
+ LivyConf.LIVY_SPARK_SCALA_VERSION.key -> getSparkScalaVersion(),
+ LivyConf.HEARTBEAT_WATCHDOG_INTERVAL.key -> "1s",
+ LivyConf.YARN_POLL_INTERVAL.key -> "500ms",
+ LivyConf.RECOVERY_MODE.key -> "recovery",
+ LivyConf.RECOVERY_STATE_STORE.key -> "filesystem",
+ LivyConf.RECOVERY_STATE_STORE_URL.key -> s"file://$configPath/state-store")
+
+ if (Cluster.isRunningOnTravis) {
+ livyConf ++= Map("livy.server.yarn.app-lookup-timeout" -> "2m")
+ }
+
+ saveProperties(livyConf, new File(configPath + "/livy.conf"))
+
+ val server = new LivyServer()
+ server.start()
+ server.livyConf.set(LivyConf.ENABLE_HIVE_CONTEXT, true)
+ // Write a serverUrl.conf file to the conf directory with the location of the Livy
+ // server. Do it atomically since it's used by MiniCluster to detect when the Livy server
+ // is up and ready.
+ eventually(timeout(30 seconds), interval(1 second)) {
+ val serverUrlConf = Map("livy.server.server-url" -> server.serverUrl())
+ saveProperties(serverUrlConf, new File(configPath + "/serverUrl.conf"))
+ }
+ }
+}
+
+private case class ProcessInfo(process: Process, logFile: File)
+
+/**
+ * Cluster implementation that uses HDFS / YARN mini clusters running as sub-processes locally.
+ * Launching Livy through this mini cluster results in three child processes:
+ *
+ * - A HDFS mini cluster
+ * - A YARN mini cluster
+ * - The Livy server
+ *
+ * Each service will write its client configuration to a temporary directory managed by the
+ * framework, so that applications can connect to the services.
+ *
+ * TODO: add support for MiniKdc.
+ */
+class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterUtils with Logging {
+ private val tempDir = new File(s"${sys.props("java.io.tmpdir")}/livy-int-test")
+ private var sparkConfDir: File = _
+ private var _configDir: File = _
+ private var hdfs: Option[ProcessInfo] = None
+ private var yarn: Option[ProcessInfo] = None
+ private var livy: Option[ProcessInfo] = None
+ private var livyUrl: String = _
+ private var _hdfsScrathDir: Path = _
+
+ override def configDir(): File = _configDir
+
+ override def hdfsScratchDir(): Path = _hdfsScrathDir
+
+ override def isRealSpark(): Boolean = {
+ new File(sys.env("SPARK_HOME") + File.separator + "RELEASE").isFile()
+ }
+
+ override def hasSparkR(): Boolean = {
+ val path = Seq(sys.env("SPARK_HOME"), "R", "lib", "sparkr.zip").mkString(File.separator)
+ new File(path).isFile()
+ }
+
+ override def doAsClusterUser[T](task: => T): T = task
+
+ // Explicitly remove the "test-lib" dependency from the classpath of child processes. We
+ // want tests to explicitly upload this jar when necessary, to test those code paths.
+ private val childClasspath = {
+ val cp = sys.props("java.class.path").split(File.pathSeparator)
+ val filtered = cp.filter { path => !new File(path).getName().startsWith("livy-test-lib-") }
+ assert(cp.size != filtered.size, "livy-test-lib jar not found in classpath!")
+ filtered.mkString(File.pathSeparator)
+ }
+
+ override def deploy(): Unit = {
+ if (tempDir.exists()) {
+ FileUtils.deleteQuietly(tempDir)
+ }
+ assert(tempDir.mkdir(), "Cannot create temp test dir.")
+ sparkConfDir = mkdir("spark-conf")
+
+ // When running a real Spark cluster, don't set the classpath.
+ val extraCp = if (!isRealSpark()) {
+ val sparkScalaVersion = getSparkScalaVersion()
+ val classPathFile =
+ new File(s"minicluster-dependencies/scala-$sparkScalaVersion/target/classpath")
+ assert(classPathFile.isFile,
+ s"Cannot read MiniCluster classpath file: ${classPathFile.getCanonicalPath}")
+ val sparkClassPath =
+ FileUtils.readFileToString(classPathFile, Charset.defaultCharset())
+
+ val dummyJar = Files.createTempFile(Paths.get(tempDir.toURI), "dummy", "jar").toFile
+ Map(
+ SparkLauncher.DRIVER_EXTRA_CLASSPATH -> sparkClassPath,
+ SparkLauncher.EXECUTOR_EXTRA_CLASSPATH -> sparkClassPath,
+ // Used for Spark 2.0. Spark 2.0 will upload specified jars to distributed cache in yarn
+ // mode, if not specified it will check jars folder. Here since jars folder is not
+ // existed, so it will throw exception.
+ "spark.yarn.jars" -> dummyJar.getAbsolutePath)
+ } else {
+ Map()
+ }
+
+ val sparkConf = extraCp ++ Map(
+ "spark.executor.instances" -> "1",
+ "spark.scheduler.minRegisteredResourcesRatio" -> "0.0",
+ "spark.ui.enabled" -> "false",
+ SparkLauncher.DRIVER_MEMORY -> "512m",
+ SparkLauncher.EXECUTOR_MEMORY -> "512m",
+ SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS -> "-Dtest.appender=console",
+ SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS -> "-Dtest.appender=console"
+ )
+ saveProperties(sparkConf, new File(sparkConfDir, "spark-defaults.conf"))
+
+ _configDir = mkdir("hadoop-conf")
+ saveProperties(config, new File(configDir, "cluster.conf"))
+ hdfs = Some(start(MiniHdfsMain.getClass, new File(configDir, "core-site.xml")))
+ yarn = Some(start(MiniYarnMain.getClass, new File(configDir, "yarn-site.xml")))
+ runLivy()
+
+ _hdfsScrathDir = fs.makeQualified(new Path("/"))
+ }
+
+ override def cleanUp(): Unit = {
+ Seq(hdfs, yarn, livy).flatten.foreach(stop)
+ hdfs = None
+ yarn = None
+ livy = None
+ }
+
+ def runLivy(): Unit = {
+ assert(!livy.isDefined)
+ val confFile = new File(configDir, "serverUrl.conf")
+ val jacocoArgs = Option(TestUtils.getJacocoArgs())
+ .map { args =>
+ Seq(args, s"-Djacoco.args=$args")
+ }.getOrElse(Nil)
+ val localLivy = start(MiniLivyMain.getClass, confFile, extraJavaArgs = jacocoArgs)
+
+ val props = loadProperties(confFile)
+ livyUrl = props("livy.server.server-url")
+
+ // Wait until Livy server responds.
+ val httpClient = new AsyncHttpClient()
+ eventually(timeout(30 seconds), interval(1 second)) {
+ val res = httpClient.prepareGet(livyUrl + "/metrics").execute().get()
+ assert(res.getStatusCode() == HttpServletResponse.SC_OK)
+ }
+
+ livy = Some(localLivy)
+ }
+
+ def stopLivy(): Unit = {
+ assert(livy.isDefined)
+ livy.foreach(stop)
+ livyUrl = null
+ livy = None
+ }
+
+ def livyEndpoint: String = livyUrl
+
+ private def mkdir(name: String, parent: File = tempDir): File = {
+ val dir = new File(parent, name)
+ if (!dir.exists()) {
+ assert(dir.mkdir(), s"Failed to create directory $name.")
+ }
+ dir
+ }
+
+ private def start(
+ klass: Class[_],
+ configFile: File,
+ extraJavaArgs: Seq[String] = Nil): ProcessInfo = {
+ val simpleName = klass.getSimpleName().stripSuffix("$")
+ val procDir = mkdir(simpleName)
+ val procTmp = mkdir("tmp", parent = procDir)
+
+ // Before starting anything, clean up previous running sessions.
+ sys.process.Process(s"pkill -f $simpleName") !
+
+ val java = sys.props("java.home") + "/bin/java"
+ val cmd =
+ Seq(
+ sys.props("java.home") + "/bin/java",
+ "-Dtest.appender=console",
+ "-Djava.io.tmpdir=" + procTmp.getAbsolutePath(),
+ "-cp", childClasspath + File.pathSeparator + configDir.getAbsolutePath(),
+ "-XX:MaxPermSize=256m") ++
+ extraJavaArgs ++
+ Seq(
+ klass.getName().stripSuffix("$"),
+ configDir.getAbsolutePath())
+
+ val logFile = new File(procDir, "output.log")
+ val pb = new ProcessBuilder(cmd.toArray: _*)
+ .directory(procDir)
+ .redirectErrorStream(true)
+ .redirectOutput(ProcessBuilder.Redirect.appendTo(logFile))
+
+ pb.environment().put("LIVY_CONF_DIR", configDir.getAbsolutePath())
+ pb.environment().put("HADOOP_CONF_DIR", configDir.getAbsolutePath())
+ pb.environment().put("SPARK_CONF_DIR", sparkConfDir.getAbsolutePath())
+ pb.environment().put("SPARK_LOCAL_IP", "127.0.0.1")
+
+ val child = pb.start()
+
+ // Wait for the config file to show up before returning, so that dependent services
+ // can see the configuration. Exit early if process dies.
+ eventually(timeout(30 seconds), interval(100 millis)) {
+ assert(configFile.isFile(), s"$simpleName hasn't started yet.")
+
+ try {
+ val exitCode = child.exitValue()
+ throw new IOException(s"Child process exited unexpectedly (exit code $exitCode)")
+ } catch {
+ case _: IllegalThreadStateException => // Try again.
+ }
+ }
+
+ ProcessInfo(child, logFile)
+ }
+
+ private def stop(svc: ProcessInfo): Unit = {
+ svc.process.destroy()
+ svc.process.waitFor()
+ }
+
+}
[23/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/JobContextImpl.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/JobContextImpl.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/JobContextImpl.java
deleted file mode 100644
index de2f6a4..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/JobContextImpl.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.driver;
-
-import java.io.File;
-import java.lang.reflect.Method;
-
-import org.apache.spark.SparkContext;
-import org.apache.spark.api.java.JavaFutureAction;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
-import org.apache.spark.sql.hive.HiveContext;
-import org.apache.spark.streaming.Duration;
-import org.apache.spark.streaming.api.java.JavaStreamingContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.JobContext;
-import com.cloudera.livy.rsc.Utils;
-
-class JobContextImpl implements JobContext {
-
- private static final Logger LOG = LoggerFactory.getLogger(JobContextImpl.class);
-
- private final JavaSparkContext sc;
- private final File localTmpDir;
- private volatile SQLContext sqlctx;
- private volatile HiveContext hivectx;
- private volatile JavaStreamingContext streamingctx;
- private final RSCDriver driver;
- private volatile Object sparksession;
-
- public JobContextImpl(JavaSparkContext sc, File localTmpDir, RSCDriver driver) {
- this.sc = sc;
- this.localTmpDir = localTmpDir;
- this.driver = driver;
- }
-
- @Override
- public JavaSparkContext sc() {
- return sc;
- }
-
- @Override
- public Object sparkSession() throws Exception {
- if (sparksession == null) {
- synchronized (this) {
- if (sparksession == null) {
- try {
- Class<?> clz = Class.forName("org.apache.spark.sql.SparkSession$");
- Object spark = clz.getField("MODULE$").get(null);
- Method m = clz.getMethod("builder");
- Object builder = m.invoke(spark);
- builder.getClass().getMethod("sparkContext", SparkContext.class)
- .invoke(builder, sc.sc());
- sparksession = builder.getClass().getMethod("getOrCreate").invoke(builder);
- } catch (Exception e) {
- LOG.warn("SparkSession is not supported", e);
- throw e;
- }
- }
- }
- }
-
- return sparksession;
- }
-
- @Override
- public SQLContext sqlctx() {
- if (sqlctx == null) {
- synchronized (this) {
- if (sqlctx == null) {
- sqlctx = new SQLContext(sc);
- }
- }
- }
- return sqlctx;
- }
-
- @Override
- public HiveContext hivectx() {
- if (hivectx == null) {
- synchronized (this) {
- if (hivectx == null) {
- hivectx = new HiveContext(sc.sc());
- }
- }
- }
- return hivectx;
- }
-
- @Override
- public synchronized JavaStreamingContext streamingctx(){
- Utils.checkState(streamingctx != null, "method createStreamingContext must be called first.");
- return streamingctx;
- }
-
- @Override
- public synchronized void createStreamingContext(long batchDuration) {
- Utils.checkState(streamingctx == null, "Streaming context is not null.");
- streamingctx = new JavaStreamingContext(sc, new Duration(batchDuration));
- }
-
- @Override
- public synchronized void stopStreamingCtx() {
- Utils.checkState(streamingctx != null, "Streaming Context is null");
- streamingctx.stop();
- streamingctx = null;
- }
-
- @Override
- public File getLocalTmpDir() {
- return localTmpDir;
- }
-
- public synchronized void stop() {
- if (streamingctx != null) {
- stopStreamingCtx();
- }
- if (sc != null) {
- sc.stop();
- }
- }
-
- public void addFile(String path) {
- driver.addFile(path);
- }
-
- public void addJarOrPyFile(String path) throws Exception {
- driver.addJarOrPyFile(path);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/JobWrapper.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/JobWrapper.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/JobWrapper.java
deleted file mode 100644
index bb24e6b..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/JobWrapper.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.driver;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.spark.api.java.JavaFutureAction;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.Job;
-
-public class JobWrapper<T> implements Callable<Void> {
-
- private static final Logger LOG = LoggerFactory.getLogger(JobWrapper.class);
-
- public final String jobId;
-
- private final RSCDriver driver;
- private final Job<T> job;
- private final AtomicInteger completed;
-
- private Future<?> future;
-
- public JobWrapper(RSCDriver driver, String jobId, Job<T> job) {
- this.driver = driver;
- this.jobId = jobId;
- this.job = job;
- this.completed = new AtomicInteger();
- }
-
- @Override
- public Void call() throws Exception {
- try {
- jobStarted();
- T result = job.call(driver.jobContext());
- finished(result, null);
- } catch (Throwable t) {
- // Catch throwables in a best-effort to report job status back to the client. It's
- // re-thrown so that the executor can destroy the affected thread (or the JVM can
- // die or whatever would happen if the throwable bubbled up).
- LOG.info("Failed to run job " + jobId, t);
- finished(null, t);
- throw new ExecutionException(t);
- } finally {
- driver.activeJobs.remove(jobId);
- }
- return null;
- }
-
- void submit(ExecutorService executor) {
- this.future = executor.submit(this);
- }
-
- void jobDone() {
- synchronized (completed) {
- completed.incrementAndGet();
- completed.notifyAll();
- }
- }
-
- boolean cancel() {
- return future != null ? future.cancel(true) : true;
- }
-
- protected void finished(T result, Throwable error) {
- if (error == null) {
- driver.jobFinished(jobId, result, null);
- } else {
- driver.jobFinished(jobId, null, error);
- }
- }
-
- protected void jobStarted() {
- driver.jobStarted(jobId);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/MutableClassLoader.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/MutableClassLoader.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/MutableClassLoader.java
deleted file mode 100644
index 772879f..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/MutableClassLoader.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.driver;
-
-import java.net.URL;
-import java.net.URLClassLoader;
-
-class MutableClassLoader extends URLClassLoader {
-
- MutableClassLoader(ClassLoader parent) {
- super(new URL[] { }, parent);
- }
-
- @Override
- public void addURL(URL url) {
- super.addURL(url);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/RSCDriver.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/RSCDriver.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/RSCDriver.java
deleted file mode 100644
index ce567b4..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/RSCDriver.java
+++ /dev/null
@@ -1,510 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.driver;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.nio.file.Files;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.PosixFilePermissions;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedDeque;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicReference;
-
-import io.netty.channel.ChannelHandler.Sharable;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.util.concurrent.ScheduledFuture;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.client.common.Serializer;
-import com.cloudera.livy.rsc.BaseProtocol;
-import com.cloudera.livy.rsc.BypassJobStatus;
-import com.cloudera.livy.rsc.FutureListener;
-import com.cloudera.livy.rsc.RSCConf;
-import com.cloudera.livy.rsc.Utils;
-import com.cloudera.livy.rsc.rpc.Rpc;
-import com.cloudera.livy.rsc.rpc.RpcDispatcher;
-import com.cloudera.livy.rsc.rpc.RpcServer;
-
-import static com.cloudera.livy.rsc.RSCConf.Entry.*;
-
-/**
- * Driver code for the Spark client library.
- */
-@Sharable
-public class RSCDriver extends BaseProtocol {
-
- private static final Logger LOG = LoggerFactory.getLogger(RSCDriver.class);
-
- private final Serializer serializer;
- private final Object jcLock;
- private final Object shutdownLock;
- private final ExecutorService executor;
- private final File localTmpDir;
- // Used to queue up requests while the SparkContext is being created.
- private final List<JobWrapper<?>> jobQueue;
- // Keeps track of connected clients.
- protected final Collection<Rpc> clients;
-
- final Map<String, JobWrapper<?>> activeJobs;
- private final Collection<BypassJobWrapper> bypassJobs;
-
- private RpcServer server;
- private volatile JobContextImpl jc;
- private volatile boolean running;
-
- protected final SparkConf conf;
- protected final RSCConf livyConf;
-
- private final AtomicReference<ScheduledFuture<?>> idleTimeout;
-
- public RSCDriver(SparkConf conf, RSCConf livyConf) throws Exception {
- Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwx------");
- this.localTmpDir = Files.createTempDirectory("rsc-tmp",
- PosixFilePermissions.asFileAttribute(perms)).toFile();
- this.executor = Executors.newCachedThreadPool();
- this.jobQueue = new LinkedList<>();
- this.clients = new ConcurrentLinkedDeque<>();
- this.serializer = new Serializer();
-
- this.conf = conf;
- this.livyConf = livyConf;
- this.jcLock = new Object();
- this.shutdownLock = new Object();
-
- this.activeJobs = new ConcurrentHashMap<>();
- this.bypassJobs = new ConcurrentLinkedDeque<>();
- this.idleTimeout = new AtomicReference<>();
- }
-
- private synchronized void shutdown() {
- if (!running) {
- return;
- }
-
- running = false;
-
- // Cancel any pending jobs.
- for (JobWrapper<?> job : activeJobs.values()) {
- job.cancel();
- }
-
- try {
- shutdownContext();
- } catch (Exception e) {
- LOG.warn("Error during shutdown.", e);
- }
- try {
- shutdownServer();
- } catch (Exception e) {
- LOG.warn("Error during shutdown.", e);
- }
-
- synchronized (shutdownLock) {
- shutdownLock.notifyAll();
- }
- synchronized (jcLock) {
- jcLock.notifyAll();
- }
- }
-
- private void initializeServer() throws Exception {
- String clientId = livyConf.get(CLIENT_ID);
- Utils.checkArgument(clientId != null, "No client ID provided.");
- String secret = livyConf.get(CLIENT_SECRET);
- Utils.checkArgument(secret != null, "No secret provided.");
-
- String launcherAddress = livyConf.get(LAUNCHER_ADDRESS);
- Utils.checkArgument(launcherAddress != null, "Missing launcher address.");
- int launcherPort = livyConf.getInt(LAUNCHER_PORT);
- Utils.checkArgument(launcherPort > 0, "Missing launcher port.");
-
- LOG.info("Connecting to: {}:{}", launcherAddress, launcherPort);
-
- // We need to unset this configuration since it doesn't really apply for the driver side.
- // If the driver runs on a multi-homed machine, this can lead to issues where the Livy
- // server cannot connect to the auto-detected address, but since the driver can run anywhere
- // on the cluster, it would be tricky to solve that problem in a generic way.
- livyConf.set(RPC_SERVER_ADDRESS, null);
-
- if (livyConf.getBoolean(TEST_STUCK_START_DRIVER)) {
- // Test flag is turned on so we will just infinite loop here. It should cause
- // timeout and we should still see yarn application being cleaned up.
- LOG.info("Infinite looping as test flag TEST_STUCK_START_SESSION is turned on.");
- while(true) {
- try {
- TimeUnit.MINUTES.sleep(10);
- } catch (InterruptedException e) {
- LOG.warn("Interrupted during test sleep.", e);
- }
- }
- }
-
- // Bring up the RpcServer an register the secret provided by the Livy server as a client.
- LOG.info("Starting RPC server...");
- this.server = new RpcServer(livyConf);
- server.registerClient(clientId, secret, new RpcServer.ClientCallback() {
- @Override
- public RpcDispatcher onNewClient(Rpc client) {
- registerClient(client);
- return RSCDriver.this;
- }
-
- @Override
- public void onSaslComplete(Rpc client) {
- onClientAuthenticated(client);
- }
- });
-
- // The RPC library takes care of timing out this.
- Rpc callbackRpc = Rpc.createClient(livyConf, server.getEventLoopGroup(),
- launcherAddress, launcherPort, clientId, secret, this).get();
- try {
- callbackRpc.call(new RemoteDriverAddress(server.getAddress(), server.getPort())).get(
- livyConf.getTimeAsMs(RPC_CLIENT_HANDSHAKE_TIMEOUT), TimeUnit.MILLISECONDS);
- } catch (TimeoutException te) {
- LOG.warn("Timed out sending address to Livy server, shutting down.");
- throw te;
- } finally {
- callbackRpc.close();
- }
-
- // At this point we install the idle timeout handler, in case the Livy server fails to connect
- // back.
- setupIdleTimeout();
- }
-
- private void registerClient(final Rpc client) {
- clients.add(client);
- stopIdleTimeout();
-
- Utils.addListener(client.getChannel().closeFuture(), new FutureListener<Void>() {
- @Override
- public void onSuccess(Void unused) {
- clients.remove(client);
- setupIdleTimeout();
- }
- });
- LOG.debug("Registered new connection from {}.", client.getChannel());
- }
-
- private void setupIdleTimeout() {
- if (clients.size() > 0) {
- return;
- }
-
- Runnable timeoutTask = new Runnable() {
- @Override
- public void run() {
- LOG.warn("Shutting down RSC due to idle timeout ({}).", livyConf.get(SERVER_IDLE_TIMEOUT));
- shutdown();
- }
- };
- ScheduledFuture<?> timeout = server.getEventLoopGroup().schedule(timeoutTask,
- livyConf.getTimeAsMs(SERVER_IDLE_TIMEOUT), TimeUnit.MILLISECONDS);
-
- // If there's already an idle task registered, then cancel the new one.
- if (!this.idleTimeout.compareAndSet(null, timeout)) {
- LOG.debug("Timeout task already registered.");
- timeout.cancel(false);
- }
-
- // If a new client connected while the idle task was being set up, then stop the task.
- if (clients.size() > 0) {
- stopIdleTimeout();
- }
- }
-
- private void stopIdleTimeout() {
- ScheduledFuture<?> idleTimeout = this.idleTimeout.getAndSet(null);
- if (idleTimeout != null) {
- LOG.debug("Cancelling idle timeout since new client connected.");
- idleTimeout.cancel(false);
- }
- }
-
- protected void broadcast(Object msg) {
- for (Rpc client : clients) {
- try {
- client.call(msg);
- } catch (Exception e) {
- LOG.warn("Failed to send message to client " + client, e);
- }
- }
- }
-
- /**
- * Initializes the SparkContext used by this driver. This implementation creates a
- * context with the provided configuration. Subclasses can override this behavior,
- * and returning a null context is allowed. In that case, the context exposed by
- * JobContext will be null.
- */
- protected JavaSparkContext initializeContext() throws Exception {
- long t1 = System.nanoTime();
- LOG.info("Starting Spark context...");
- JavaSparkContext sc = new JavaSparkContext(conf);
- LOG.info("Spark context finished initialization in {}ms",
- TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t1));
- return sc;
- }
-
- protected void onClientAuthenticated(final Rpc client) {
-
- }
-
- /**
- * Called to shut down the driver; any initialization done by initializeContext() should
- * be undone here. This is guaranteed to be called only once.
- */
- protected void shutdownContext() {
- if (jc != null) {
- jc.stop();
- }
- executor.shutdownNow();
- try {
- FileUtils.deleteDirectory(localTmpDir);
- } catch (IOException e) {
- LOG.warn("Failed to delete local tmp dir: " + localTmpDir, e);
- }
- }
-
- private void shutdownServer() {
- if (server != null) {
- server.close();
- }
- for (Rpc client: clients) {
- client.close();
- }
- }
-
- void run() throws Exception {
- this.running = true;
-
- // Set up a class loader that can be modified, so that we can add jars uploaded
- // by the client to the driver's class path.
- ClassLoader driverClassLoader = new MutableClassLoader(
- Thread.currentThread().getContextClassLoader());
- Thread.currentThread().setContextClassLoader(driverClassLoader);
-
- try {
- initializeServer();
-
- JavaSparkContext sc = initializeContext();
- synchronized (jcLock) {
- jc = new JobContextImpl(sc, localTmpDir, this);
- jcLock.notifyAll();
- }
-
- synchronized (jcLock) {
- for (JobWrapper<?> job : jobQueue) {
- submit(job);
- }
- jobQueue.clear();
- }
-
- synchronized (shutdownLock) {
- try {
- while (running) {
- shutdownLock.wait();
- }
- } catch (InterruptedException ie) {
- // Nothing to do.
- }
- }
- } finally {
- shutdown();
- }
- }
-
- public void submit(JobWrapper<?> job) {
- if (jc != null) {
- job.submit(executor);
- return;
- }
- synchronized (jcLock) {
- if (jc != null) {
- job.submit(executor);
- } else {
- LOG.info("SparkContext not yet up, queueing job request.");
- jobQueue.add(job);
- }
- }
- }
-
- JobContextImpl jobContext() {
- return jc;
- }
-
- Serializer serializer() {
- return serializer;
- }
-
- <T> void jobFinished(String jobId, T result, Throwable error) {
- LOG.debug("Send job({}) result to Client.", jobId);
- broadcast(new JobResult<T>(jobId, result, error));
- }
-
- void jobStarted(String jobId) {
- broadcast(new JobStarted(jobId));
- }
-
- public void handle(ChannelHandlerContext ctx, CancelJob msg) {
- JobWrapper<?> job = activeJobs.get(msg.id);
- if (job == null || !job.cancel()) {
- LOG.info("Requested to cancel an already finished job.");
- }
- }
-
- public void handle(ChannelHandlerContext ctx, EndSession msg) {
- if (livyConf.getBoolean(TEST_STUCK_END_SESSION)) {
- LOG.warn("Ignoring EndSession request because TEST_STUCK_END_SESSION is set.");
- } else {
- LOG.debug("Shutting down due to EndSession request.");
- shutdown();
- }
- }
-
- public void handle(ChannelHandlerContext ctx, JobRequest<?> msg) {
- LOG.info("Received job request {}", msg.id);
- JobWrapper<?> wrapper = new JobWrapper<>(this, msg.id, msg.job);
- activeJobs.put(msg.id, wrapper);
- submit(wrapper);
- }
-
- public void handle(ChannelHandlerContext ctx, BypassJobRequest msg) throws Exception {
- LOG.info("Received bypass job request {}", msg.id);
- BypassJobWrapper wrapper = createWrapper(msg);
- bypassJobs.add(wrapper);
- activeJobs.put(msg.id, wrapper);
- if (msg.synchronous) {
- waitForJobContext();
- try {
- wrapper.call();
- } catch (Throwable t) {
- // Wrapper already logged and saved the exception, just avoid it bubbling up
- // to the RPC layer.
- }
- } else {
- submit(wrapper);
- }
- }
-
- protected BypassJobWrapper createWrapper(BypassJobRequest msg) throws Exception {
- return new BypassJobWrapper(this, msg.id, new BypassJob(this.serializer(), msg.serializedJob));
- }
-
- @SuppressWarnings("unchecked")
- public Object handle(ChannelHandlerContext ctx, SyncJobRequest msg) throws Exception {
- waitForJobContext();
- return msg.job.call(jc);
- }
-
- public BypassJobStatus handle(ChannelHandlerContext ctx, GetBypassJobStatus msg) {
- for (Iterator<BypassJobWrapper> it = bypassJobs.iterator(); it.hasNext();) {
- BypassJobWrapper job = it.next();
- if (job.jobId.equals(msg.id)) {
- BypassJobStatus status = job.getStatus();
- switch (status.state) {
- case CANCELLED:
- case FAILED:
- case SUCCEEDED:
- it.remove();
- break;
-
- default:
- // No-op.
- }
- return status;
- }
- }
-
- throw new NoSuchElementException(msg.id);
- }
-
- private void waitForJobContext() throws InterruptedException {
- synchronized (jcLock) {
- while (jc == null) {
- jcLock.wait();
- if (!running) {
- throw new IllegalStateException("Remote context is shutting down.");
- }
- }
- }
- }
-
- protected void addFile(String path) {
- jc.sc().addFile(path);
- }
-
- protected void addJarOrPyFile(String path) throws Exception {
- File localCopyDir = new File(jc.getLocalTmpDir(), "__livy__");
- File localCopy = copyFileToLocal(localCopyDir, path, jc.sc().sc());
- addLocalFileToClassLoader(localCopy);
- jc.sc().addJar(path);
- }
-
- public void addLocalFileToClassLoader(File localCopy) throws MalformedURLException {
- MutableClassLoader cl = (MutableClassLoader) Thread.currentThread().getContextClassLoader();
- cl.addURL(localCopy.toURI().toURL());
- }
-
- public File copyFileToLocal(
- File localCopyDir,
- String filePath,
- SparkContext sc) throws Exception {
- synchronized (jc) {
- if (!localCopyDir.isDirectory() && !localCopyDir.mkdir()) {
- throw new IOException("Failed to create directory to add pyFile");
- }
- }
- URI uri = new URI(filePath);
- String name = uri.getFragment() != null ? uri.getFragment() : uri.getPath();
- name = new File(name).getName();
- File localCopy = new File(localCopyDir, name);
-
- if (localCopy.exists()) {
- throw new IOException(String.format("A file with name %s has " +
- "already been uploaded.", name));
- }
- Configuration conf = sc.hadoopConfiguration();
- FileSystem fs = FileSystem.get(uri, conf);
- fs.copyToLocalFile(new Path(uri), new Path(localCopy.toURI()));
- return localCopy;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/RSCDriverBootstrapper.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/RSCDriverBootstrapper.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/RSCDriverBootstrapper.java
deleted file mode 100644
index 4c326c2..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/RSCDriverBootstrapper.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.driver;
-
-import java.io.FileInputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.util.Properties;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import org.apache.spark.SparkConf;
-
-import com.cloudera.livy.rsc.RSCConf;
-import static com.cloudera.livy.rsc.RSCConf.Entry.*;
-
-/**
- * The entry point for the RSC. Parses command line arguments and instantiates the correct
- * driver based on the configuration.
- *
- * The driver is expected to have a public constructor that takes a two parameters:
- * a SparkConf and a RSCConf.
- */
-public final class RSCDriverBootstrapper {
-
- public static void main(String[] args) throws Exception {
- Properties props;
-
- switch (args.length) {
- case 0:
- props = System.getProperties();
- break;
-
- case 1:
- props = new Properties();
- Reader r = new InputStreamReader(new FileInputStream(args[0]), UTF_8);
- try {
- props.load(r);
- } finally {
- r.close();
- }
- break;
-
- default:
- throw new IllegalArgumentException("Too many arguments.");
- }
-
- SparkConf conf = new SparkConf(false);
- RSCConf livyConf = new RSCConf(null);
-
- for (String key : props.stringPropertyNames()) {
- String value = props.getProperty(key);
- if (key.startsWith(RSCConf.LIVY_SPARK_PREFIX)) {
- livyConf.set(key.substring(RSCConf.LIVY_SPARK_PREFIX.length()), value);
- props.remove(key);
- } else if (key.startsWith(RSCConf.SPARK_CONF_PREFIX)) {
- conf.set(key, value);
- }
- }
-
- String driverClass = livyConf.get(DRIVER_CLASS);
- if (driverClass == null) {
- driverClass = RSCDriver.class.getName();
- }
-
- RSCDriver driver = (RSCDriver) Thread.currentThread()
- .getContextClassLoader()
- .loadClass(driverClass)
- .getConstructor(SparkConf.class, RSCConf.class)
- .newInstance(conf, livyConf);
-
- driver.run();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/Statement.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/Statement.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/Statement.java
deleted file mode 100644
index 67aaa1f..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/Statement.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.driver;
-
-import java.util.concurrent.atomic.AtomicReference;
-
-import com.fasterxml.jackson.annotation.JsonRawValue;
-
-public class Statement {
- public final Integer id;
- public final String code;
- public final AtomicReference<StatementState> state;
- @JsonRawValue
- public volatile String output;
- public double progress;
-
- public Statement(Integer id, String code, StatementState state, String output) {
- this.id = id;
- this.code = code;
- this.state = new AtomicReference<>(state);
- this.output = output;
- this.progress = 0.0;
- }
-
- public Statement() {
- this(null, null, null, null);
- }
-
- public boolean compareAndTransit(final StatementState from, final StatementState to) {
- if (state.compareAndSet(from, to)) {
- StatementState.validate(from, to);
- return true;
- }
- return false;
- }
-
- public void updateProgress(double p) {
- if (this.state.get().isOneOf(StatementState.Cancelled, StatementState.Available)) {
- this.progress = 1.0;
- } else {
- this.progress = p;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/driver/StatementState.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/driver/StatementState.java b/rsc/src/main/java/com/cloudera/livy/rsc/driver/StatementState.java
deleted file mode 100644
index 61a86d0..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/driver/StatementState.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.driver;
-
-import java.util.*;
-
-import com.fasterxml.jackson.annotation.JsonValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public enum StatementState {
- Waiting("waiting"),
- Running("running"),
- Available("available"),
- Cancelling("cancelling"),
- Cancelled("cancelled");
-
- private static final Logger LOG = LoggerFactory.getLogger(StatementState.class);
-
- private final String state;
-
- StatementState(final String text) {
- this.state = text;
- }
-
- @JsonValue
- @Override
- public String toString() {
- return state;
- }
-
- public boolean isOneOf(StatementState... states) {
- for (StatementState s : states) {
- if (s == this) {
- return true;
- }
- }
- return false;
- }
-
- private static final Map<StatementState, List<StatementState>> PREDECESSORS;
-
- static void put(StatementState key,
- Map<StatementState, List<StatementState>> map,
- StatementState... values) {
- map.put(key, Collections.unmodifiableList(Arrays.asList(values)));
- }
-
- static {
- final Map<StatementState, List<StatementState>> predecessors =
- new EnumMap<>(StatementState.class);
- put(Waiting, predecessors);
- put(Running, predecessors, Waiting);
- put(Available, predecessors, Running);
- put(Cancelling, predecessors, Running);
- put(Cancelled, predecessors, Waiting, Cancelling);
-
- PREDECESSORS = Collections.unmodifiableMap(predecessors);
- }
-
- static boolean isValid(StatementState from, StatementState to) {
- return PREDECESSORS.get(to).contains(from);
- }
-
- static void validate(StatementState from, StatementState to) {
- LOG.debug("{} -> {}", from, to);
- if (!isValid(from, to)) {
- throw new IllegalStateException("Illegal Transition: " + from + " -> " + to);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/rpc/KryoMessageCodec.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/KryoMessageCodec.java b/rsc/src/main/java/com/cloudera/livy/rsc/rpc/KryoMessageCodec.java
deleted file mode 100644
index b784160..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/KryoMessageCodec.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.rpc;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.ByteBufferInputStream;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import com.esotericsoftware.shaded.org.objenesis.strategy.StdInstantiatorStrategy;
-import io.netty.buffer.ByteBuf;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.ByteToMessageCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.client.common.Serializer;
-import com.cloudera.livy.rsc.Utils;
-
-/**
- * Codec that serializes / deserializes objects using Kryo. Objects are encoded with a 4-byte
- * header with the length of the serialized data.
- */
-class KryoMessageCodec extends ByteToMessageCodec<Object> {
-
- private static final Logger LOG = LoggerFactory.getLogger(KryoMessageCodec.class);
-
- private final int maxMessageSize;
- private final Serializer serializer;
- private volatile EncryptionHandler encryptionHandler;
-
- public KryoMessageCodec(int maxMessageSize, Class<?>... messages) {
- this.maxMessageSize = maxMessageSize;
- this.serializer = new Serializer(messages);
- this.encryptionHandler = null;
- }
-
- @Override
- protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out)
- throws Exception {
- if (in.readableBytes() < 4) {
- return;
- }
-
- in.markReaderIndex();
- int msgSize = in.readInt();
- checkSize(msgSize);
-
- if (in.readableBytes() < msgSize) {
- // Incomplete message in buffer.
- in.resetReaderIndex();
- return;
- }
-
- try {
- ByteBuffer nioBuffer = maybeDecrypt(in.nioBuffer(in.readerIndex(), msgSize));
- Object msg = serializer.deserialize(nioBuffer);
- LOG.debug("Decoded message of type {} ({} bytes)",
- msg != null ? msg.getClass().getName() : msg, msgSize);
- out.add(msg);
- } finally {
- in.skipBytes(msgSize);
- }
- }
-
- @Override
- protected void encode(ChannelHandlerContext ctx, Object msg, ByteBuf buf)
- throws Exception {
- ByteBuffer msgData = maybeEncrypt(serializer.serialize(msg));
- LOG.debug("Encoded message of type {} ({} bytes)", msg.getClass().getName(),
- msgData.remaining());
- checkSize(msgData.remaining());
-
- buf.ensureWritable(msgData.remaining() + 4);
- buf.writeInt(msgData.remaining());
- buf.writeBytes(msgData);
- }
-
- @Override
- public void channelInactive(ChannelHandlerContext ctx) throws Exception {
- if (encryptionHandler != null) {
- encryptionHandler.dispose();
- }
- super.channelInactive(ctx);
- }
-
- private void checkSize(int msgSize) {
- Utils.checkArgument(msgSize > 0, "Message size (%s bytes) must be positive.", msgSize);
- Utils.checkArgument(maxMessageSize <= 0 || msgSize <= maxMessageSize,
- "Message (%s bytes) exceeds maximum allowed size (%s bytes).", msgSize, maxMessageSize);
- }
-
- private ByteBuffer maybeEncrypt(ByteBuffer data) throws Exception {
- return doWrapOrUnWrap(data, true);
- }
-
- private ByteBuffer maybeDecrypt(ByteBuffer data) throws Exception {
- return doWrapOrUnWrap(data, false);
- }
-
- private ByteBuffer doWrapOrUnWrap(ByteBuffer data, boolean wrap) throws IOException {
- if (encryptionHandler == null) {
- return data;
- }
-
- byte[] byteData;
- int len = data.limit() - data.position();
- int offset;
- if (data.hasArray()) {
- byteData = data.array();
- offset = data.position() + data.arrayOffset();
- data.position(data.limit());
- } else {
- byteData = new byte[len];
- offset = 0;
- data.get(byteData);
- }
-
- byte[] result;
- if (wrap) {
- result = encryptionHandler.wrap(byteData, offset, len);
- } else {
- result = encryptionHandler.unwrap(byteData, offset, len);
- }
- return ByteBuffer.wrap(result);
- }
-
- void setEncryptionHandler(EncryptionHandler handler) {
- this.encryptionHandler = handler;
- }
-
- interface EncryptionHandler {
-
- byte[] wrap(byte[] data, int offset, int len) throws IOException;
-
- byte[] unwrap(byte[] data, int offset, int len) throws IOException;
-
- void dispose() throws IOException;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/rpc/README.md
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/README.md b/rsc/src/main/java/com/cloudera/livy/rsc/rpc/README.md
deleted file mode 100644
index ca9e98c..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/README.md
+++ /dev/null
@@ -1,32 +0,0 @@
-Spark Client RPC
-================
-
-Basic flow of events:
-
-- Client side creates an RPC server
-- Client side spawns RemoteDriver, which manages the SparkContext, and provides a secret
-- Client side sets up a timer to wait for RemoteDriver to connect back
-- RemoteDriver connects back to client, SASL handshake ensues
-- Connection is established and now there's a session between the client and the driver.
-
-Features of the RPC layer:
-
-- All messages serialized via Kryo
-- All messages are replied to. It's either an empty "ack" or an actual response - that depends
- on the message.
-- RPC send API is asynchronous - callers get a future that can be used to wait for the message.
-- Currently, no connection retry. If connection goes down, both sides tear down the session.
-
-Notes:
-
-- Because serialization is using Kryo, types need explicit empty constructors or things will
- fail to deserialize. This can be seen in the way exceptions are propagated - the throwing
- side sends just a string stack trace to the remote, because certain fields on exceptions
- don't have empty constructors.
-- The above is especially important because at the moment there's no way to register custom
- serializers in the RPC library.
-
-Future work:
-
-- Random initial RPC id + id wrapping.
-- SSL / security in general.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/rpc/Rpc.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/Rpc.java b/rsc/src/main/java/com/cloudera/livy/rsc/rpc/Rpc.java
deleted file mode 100644
index f095eff..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/Rpc.java
+++ /dev/null
@@ -1,460 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.rpc;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import javax.security.auth.callback.Callback;
-import javax.security.auth.callback.CallbackHandler;
-import javax.security.auth.callback.NameCallback;
-import javax.security.auth.callback.PasswordCallback;
-import javax.security.sasl.RealmCallback;
-import javax.security.sasl.Sasl;
-import javax.security.sasl.SaslClient;
-import javax.security.sasl.SaslException;
-
-import io.netty.bootstrap.Bootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInboundHandlerAdapter;
-import io.netty.channel.ChannelOption;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.embedded.EmbeddedChannel;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioSocketChannel;
-import io.netty.handler.logging.LogLevel;
-import io.netty.handler.logging.LoggingHandler;
-import io.netty.util.concurrent.EventExecutorGroup;
-import io.netty.util.concurrent.Future;
-import io.netty.util.concurrent.GenericFutureListener;
-import io.netty.util.concurrent.ImmediateEventExecutor;
-import io.netty.util.concurrent.Promise;
-import io.netty.util.concurrent.ScheduledFuture;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.rsc.RSCConf;
-import com.cloudera.livy.rsc.Utils;
-import static com.cloudera.livy.rsc.RSCConf.Entry.*;
-
-/**
- * Encapsulates the RPC functionality. Provides higher-level methods to talk to the remote
- * endpoint.
- */
-public class Rpc implements Closeable {
-
- private static final Logger LOG = LoggerFactory.getLogger(Rpc.class);
-
- static final String SASL_REALM = "rsc";
- static final String SASL_USER = "rsc";
- static final String SASL_PROTOCOL = "rsc";
- static final String SASL_AUTH_CONF = "auth-conf";
-
- /**
- * Creates an RPC client for a server running on the given remote host and port.
- *
- * @param config RPC configuration data.
- * @param eloop Event loop for managing the connection.
- * @param host Host name or IP address to connect to.
- * @param port Port where server is listening.
- * @param clientId The client ID that identifies the connection.
- * @param secret Secret for authenticating the client with the server.
- * @param dispatcher Dispatcher used to handle RPC calls.
- * @return A future that can be used to monitor the creation of the RPC object.
- */
- public static Promise<Rpc> createClient(
- final RSCConf config,
- final EventLoopGroup eloop,
- String host,
- int port,
- final String clientId,
- final String secret,
- final RpcDispatcher dispatcher) throws Exception {
- int connectTimeoutMs = (int) config.getTimeAsMs(RPC_CLIENT_CONNECT_TIMEOUT);
-
- final ChannelFuture cf = new Bootstrap()
- .group(eloop)
- .handler(new ChannelInboundHandlerAdapter() { })
- .channel(NioSocketChannel.class)
- .option(ChannelOption.SO_KEEPALIVE, true)
- .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, connectTimeoutMs)
- .connect(host, port);
-
- final Promise<Rpc> promise = eloop.next().newPromise();
- final AtomicReference<Rpc> rpc = new AtomicReference<Rpc>();
-
- // Set up a timeout to undo everything.
- final Runnable timeoutTask = new Runnable() {
- @Override
- public void run() {
- promise.setFailure(new TimeoutException("Timed out waiting for RPC server connection."));
- }
- };
- final ScheduledFuture<?> timeoutFuture = eloop.schedule(timeoutTask,
- config.getTimeAsMs(RPC_CLIENT_HANDSHAKE_TIMEOUT), TimeUnit.MILLISECONDS);
-
- // The channel listener instantiates the Rpc instance when the connection is established,
- // and initiates the SASL handshake.
- cf.addListener(new ChannelFutureListener() {
- @Override
- public void operationComplete(ChannelFuture cf) throws Exception {
- if (cf.isSuccess()) {
- SaslClientHandler saslHandler = new SaslClientHandler(config, clientId, promise,
- timeoutFuture, secret, dispatcher);
- Rpc rpc = createRpc(config, saslHandler, (SocketChannel) cf.channel(), eloop);
- saslHandler.rpc = rpc;
- saslHandler.sendHello(cf.channel());
- } else {
- promise.setFailure(cf.cause());
- }
- }
- });
-
- // Handle cancellation of the promise.
- promise.addListener(new GenericFutureListener<Promise<Rpc>>() {
- @Override
- public void operationComplete(Promise<Rpc> p) {
- if (p.isCancelled()) {
- cf.cancel(true);
- }
- }
- });
-
- return promise;
- }
-
- static Rpc createServer(SaslHandler saslHandler, RSCConf config, SocketChannel channel,
- EventExecutorGroup egroup) throws IOException {
- return createRpc(config, saslHandler, channel, egroup);
- }
-
- private static Rpc createRpc(RSCConf config,
- SaslHandler saslHandler,
- SocketChannel client,
- EventExecutorGroup egroup)
- throws IOException {
- LogLevel logLevel = LogLevel.TRACE;
- String logLevelStr = config.get(RPC_CHANNEL_LOG_LEVEL);
- if (logLevelStr != null) {
- try {
- logLevel = LogLevel.valueOf(logLevelStr);
- } catch (Exception e) {
- LOG.warn("Invalid log level {}, reverting to default.", logLevelStr);
- }
- }
-
- boolean logEnabled = false;
- switch (logLevel) {
- case DEBUG:
- logEnabled = LOG.isDebugEnabled();
- break;
- case ERROR:
- logEnabled = LOG.isErrorEnabled();
- break;
- case INFO:
- logEnabled = LOG.isInfoEnabled();
- break;
- case TRACE:
- logEnabled = LOG.isTraceEnabled();
- break;
- case WARN:
- logEnabled = LOG.isWarnEnabled();
- break;
- }
-
- if (logEnabled) {
- client.pipeline().addLast("logger", new LoggingHandler(Rpc.class, logLevel));
- }
-
- KryoMessageCodec kryo = new KryoMessageCodec(config.getInt(RPC_MAX_MESSAGE_SIZE),
- MessageHeader.class, NullMessage.class, SaslMessage.class);
- saslHandler.setKryoMessageCodec(kryo);
- client.pipeline()
- .addLast("codec", kryo)
- .addLast("sasl", saslHandler);
- return new Rpc(config, client, egroup);
- }
-
- static Rpc createEmbedded(RpcDispatcher dispatcher) {
- EmbeddedChannel c = new EmbeddedChannel(
- new LoggingHandler(Rpc.class),
- new KryoMessageCodec(0, MessageHeader.class, NullMessage.class),
- dispatcher);
- Rpc rpc = new Rpc(new RSCConf(null), c, ImmediateEventExecutor.INSTANCE);
- rpc.dispatcher = dispatcher;
- return rpc;
- }
-
- private final RSCConf config;
- private final AtomicBoolean rpcClosed;
- private final AtomicLong rpcId;
- private final Channel channel;
- private final EventExecutorGroup egroup;
- private volatile RpcDispatcher dispatcher;
-
- private Rpc(RSCConf config, Channel channel, EventExecutorGroup egroup) {
- Utils.checkArgument(channel != null);
- Utils.checkArgument(egroup != null);
- this.config = config;
- this.channel = channel;
- this.dispatcher = null;
- this.egroup = egroup;
- this.rpcClosed = new AtomicBoolean();
- this.rpcId = new AtomicLong();
-
- // Note: this does not work for embedded channels.
- channel.pipeline().addLast("monitor", new ChannelInboundHandlerAdapter() {
- @Override
- public void channelInactive(ChannelHandlerContext ctx) throws Exception {
- close();
- super.channelInactive(ctx);
- }
- });
- }
-
- /**
- * Send an RPC call to the remote endpoint and returns a future that can be used to monitor the
- * operation.
- */
- public Future<Void> call(Object msg) {
- return call(msg, Void.class);
- }
-
- /**
- * Send an RPC call to the remote endpoint and returns a future that can be used to monitor the
- * operation.
- *
- * @param msg RPC call to send.
- * @param retType Type of expected reply.
- * @return A future used to monitor the operation.
- */
- public <T> Future<T> call(final Object msg, Class<T> retType) {
- Utils.checkArgument(msg != null);
- Utils.checkState(channel.isOpen(), "RPC channel is closed.");
- try {
- final long id = rpcId.getAndIncrement();
- final Promise<T> promise = egroup.next().newPromise();
- final ChannelFutureListener listener = new ChannelFutureListener() {
- @Override
- public void operationComplete(ChannelFuture cf) {
- if (!cf.isSuccess() && !promise.isDone()) {
- LOG.warn("Failed to send RPC, closing connection.", cf.cause());
- promise.setFailure(cf.cause());
- dispatcher.discardRpc(id);
- close();
- }
- }
- };
-
- dispatcher.registerRpc(id, promise, msg.getClass().getName());
- channel.eventLoop().submit(new Runnable() {
- @Override
- public void run() {
- channel.write(new MessageHeader(id, Rpc.MessageType.CALL)).addListener(listener);
- channel.writeAndFlush(msg).addListener(listener);
- }
- });
-
- return promise;
- } catch (Exception e) {
- throw Utils.propagate(e);
- }
- }
-
- public Channel getChannel() {
- return channel;
- }
-
- void setDispatcher(RpcDispatcher dispatcher) {
- Utils.checkNotNull(dispatcher);
- Utils.checkState(this.dispatcher == null, "Dispatcher already set.");
- this.dispatcher = dispatcher;
- channel.pipeline().addLast("dispatcher", dispatcher);
- }
-
- @Override
- public void close() {
- if (!rpcClosed.compareAndSet(false, true)) {
- return;
- }
- try {
- channel.close().sync();
- } catch (InterruptedException ie) {
- Thread.interrupted();
- }
- }
-
- static enum MessageType {
- CALL,
- REPLY,
- ERROR;
- }
-
- static class MessageHeader {
- final long id;
- final MessageType type;
-
- MessageHeader() {
- this(-1, null);
- }
-
- MessageHeader(long id, MessageType type) {
- this.id = id;
- this.type = type;
- }
-
- }
-
- static class NullMessage {
-
- }
-
- static class SaslMessage {
- final String clientId;
- final byte[] payload;
-
- SaslMessage() {
- this(null, null);
- }
-
- SaslMessage(byte[] payload) {
- this(null, payload);
- }
-
- SaslMessage(String clientId, byte[] payload) {
- this.clientId = clientId;
- this.payload = payload;
- }
-
- }
-
- private static class SaslClientHandler extends SaslHandler implements CallbackHandler {
-
- private final SaslClient client;
- private final String clientId;
- private final String secret;
- private final RpcDispatcher dispatcher;
- private Promise<Rpc> promise;
- private ScheduledFuture<?> timeout;
-
- // Can't be set in constructor due to circular dependency.
- private Rpc rpc;
-
- SaslClientHandler(
- RSCConf config,
- String clientId,
- Promise<Rpc> promise,
- ScheduledFuture<?> timeout,
- String secret,
- RpcDispatcher dispatcher)
- throws IOException {
- super(config);
- this.clientId = clientId;
- this.promise = promise;
- this.timeout = timeout;
- this.secret = secret;
- this.dispatcher = dispatcher;
- this.client = Sasl.createSaslClient(new String[] { config.get(SASL_MECHANISMS) },
- null, SASL_PROTOCOL, SASL_REALM, config.getSaslOptions(), this);
- }
-
- @Override
- protected boolean isComplete() {
- return client.isComplete();
- }
-
- @Override
- protected String getNegotiatedProperty(String name) {
- return (String) client.getNegotiatedProperty(name);
- }
-
- @Override
- protected SaslMessage update(SaslMessage challenge) throws IOException {
- byte[] response = client.evaluateChallenge(challenge.payload);
- return response != null ? new SaslMessage(response) : null;
- }
-
- @Override
- public byte[] wrap(byte[] data, int offset, int len) throws IOException {
- return client.wrap(data, offset, len);
- }
-
- @Override
- public byte[] unwrap(byte[] data, int offset, int len) throws IOException {
- return client.unwrap(data, offset, len);
- }
-
- @Override
- public void dispose() throws IOException {
- if (!client.isComplete()) {
- onError(new SaslException("Client closed before SASL negotiation finished."));
- }
- client.dispose();
- }
-
- @Override
- protected void onComplete() throws Exception {
- timeout.cancel(true);
- rpc.setDispatcher(dispatcher);
- promise.setSuccess(rpc);
- timeout = null;
- promise = null;
- }
-
- @Override
- protected void onError(Throwable error) {
- timeout.cancel(true);
- if (!promise.isDone()) {
- promise.setFailure(error);
- }
- }
-
- @Override
- public void handle(Callback[] callbacks) {
- for (Callback cb : callbacks) {
- if (cb instanceof NameCallback) {
- ((NameCallback)cb).setName(clientId);
- } else if (cb instanceof PasswordCallback) {
- ((PasswordCallback)cb).setPassword(secret.toCharArray());
- } else if (cb instanceof RealmCallback) {
- RealmCallback rb = (RealmCallback) cb;
- rb.setText(rb.getDefaultText());
- }
- }
- }
-
- void sendHello(Channel c) throws Exception {
- byte[] hello = client.hasInitialResponse() ?
- client.evaluateChallenge(new byte[0]) : new byte[0];
- c.writeAndFlush(new SaslMessage(clientId, hello)).sync();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcDispatcher.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcDispatcher.java b/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcDispatcher.java
deleted file mode 100644
index 38f6867..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcDispatcher.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.rpc;
-
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-import io.netty.util.concurrent.Promise;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.rsc.Utils;
-
-/**
- * An implementation of ChannelInboundHandler that dispatches incoming messages to an instance
- * method based on the method signature.
- * <p/>
- * A handler's signature must be of the form:
- * <p/>
- * <blockquote><tt>protected void handle(ChannelHandlerContext, MessageType)</tt></blockquote>
- * <p/>
- * Where "MessageType" must match exactly the type of the message to handle. Polymorphism is not
- * supported. Handlers can return a value, which becomes the RPC reply; if a null is returned, then
- * a reply is still sent, with an empty payload.
- */
-public abstract class RpcDispatcher extends SimpleChannelInboundHandler<Object> {
-
- private static final Logger LOG = LoggerFactory.getLogger(RpcDispatcher.class);
-
- private final Map<Class<?>, Method> handlers = new ConcurrentHashMap<>();
- private final Collection<OutstandingRpc> rpcs = new ConcurrentLinkedQueue<OutstandingRpc>();
-
- private volatile Rpc.MessageHeader lastHeader;
-
- /** Override this to add a name to the dispatcher, for debugging purposes. */
- protected String name() {
- return getClass().getSimpleName();
- }
-
- @Override
- protected final void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception {
- if (lastHeader == null) {
- if (!(msg instanceof Rpc.MessageHeader)) {
- LOG.warn("[{}] Expected RPC header, got {} instead.", name(),
- msg != null ? msg.getClass().getName() : null);
- throw new IllegalArgumentException();
- }
- lastHeader = (Rpc.MessageHeader) msg;
- } else {
- LOG.debug("[{}] Received RPC message: type={} id={} payload={}", name(),
- lastHeader.type, lastHeader.id, msg != null ? msg.getClass().getName() : null);
- try {
- switch (lastHeader.type) {
- case CALL:
- handleCall(ctx, msg);
- break;
- case REPLY:
- handleReply(ctx, msg, findRpc(lastHeader.id));
- break;
- case ERROR:
- handleError(ctx, msg, findRpc(lastHeader.id));
- break;
- default:
- throw new IllegalArgumentException("Unknown RPC message type: " + lastHeader.type);
- }
- } finally {
- lastHeader = null;
- }
- }
- }
-
- private OutstandingRpc findRpc(long id) {
- for (Iterator<OutstandingRpc> it = rpcs.iterator(); it.hasNext();) {
- OutstandingRpc rpc = it.next();
- if (rpc.id == id) {
- it.remove();
- return rpc;
- }
- }
- throw new IllegalArgumentException(String.format(
- "Received RPC reply for unknown RPC (%d).", id));
- }
-
- private void handleCall(ChannelHandlerContext ctx, Object msg) throws Exception {
- Method handler = handlers.get(msg.getClass());
- if (handler == null) {
- // Try both getDeclaredMethod() and getMethod() so that we try both private methods
- // of the class, and public methods of parent classes.
- try {
- handler = getClass().getDeclaredMethod("handle", ChannelHandlerContext.class,
- msg.getClass());
- } catch (NoSuchMethodException e) {
- try {
- handler = getClass().getMethod("handle", ChannelHandlerContext.class,
- msg.getClass());
- } catch (NoSuchMethodException e2) {
- LOG.warn(String.format("[%s] Failed to find handler for msg '%s'.", name(),
- msg.getClass().getName()));
- writeMessage(ctx, Rpc.MessageType.ERROR, Utils.stackTraceAsString(e.getCause()));
- return;
- }
- }
- handler.setAccessible(true);
- handlers.put(msg.getClass(), handler);
- }
-
- try {
- Object payload = handler.invoke(this, ctx, msg);
- if (payload == null) {
- payload = new Rpc.NullMessage();
- }
- writeMessage(ctx, Rpc.MessageType.REPLY, payload);
- } catch (InvocationTargetException ite) {
- LOG.debug(String.format("[%s] Error in RPC handler.", name()), ite.getCause());
- writeMessage(ctx, Rpc.MessageType.ERROR, Utils.stackTraceAsString(ite.getCause()));
- }
- }
-
- private void writeMessage(ChannelHandlerContext ctx, Rpc.MessageType replyType, Object payload) {
- ctx.channel().write(new Rpc.MessageHeader(lastHeader.id, replyType));
- ctx.channel().writeAndFlush(payload);
- }
-
- private void handleReply(ChannelHandlerContext ctx, Object msg, OutstandingRpc rpc)
- throws Exception {
- rpc.future.setSuccess(msg instanceof Rpc.NullMessage ? null : msg);
- }
-
- private void handleError(ChannelHandlerContext ctx, Object msg, OutstandingRpc rpc)
- throws Exception {
- if (msg instanceof String) {
- LOG.warn("Received error message:{}.", msg);
- rpc.future.setFailure(new RpcException((String) msg));
- } else {
- String error = String.format("Received error with unexpected payload (%s).",
- msg != null ? msg.getClass().getName() : null);
- LOG.warn(String.format("[%s] %s", name(), error));
- rpc.future.setFailure(new IllegalArgumentException(error));
- ctx.close();
- }
- }
-
- @Override
- public final void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("[%s] Caught exception in channel pipeline.", name()), cause);
- } else {
- LOG.info("[{}] Closing channel due to exception in pipeline ({}).", name(),
- cause.getMessage());
- }
-
- if (lastHeader != null) {
- // There's an RPC waiting for a reply. Exception was most probably caught while processing
- // the RPC, so send an error.
- ctx.channel().write(new Rpc.MessageHeader(lastHeader.id, Rpc.MessageType.ERROR));
- ctx.channel().writeAndFlush(Utils.stackTraceAsString(cause));
- lastHeader = null;
- }
-
- ctx.close();
- }
-
- @Override
- public final void channelInactive(ChannelHandlerContext ctx) throws Exception {
- if (rpcs.size() > 0) {
- LOG.warn("[{}] Closing RPC channel with {} outstanding RPCs.", name(), rpcs.size());
- for (OutstandingRpc rpc : rpcs) {
- rpc.future.cancel(true);
- }
- } else {
- LOG.debug("Channel {} became inactive.", ctx.channel());
- }
- super.channelInactive(ctx);
- }
-
- void registerRpc(long id, Promise<?> promise, String type) {
- LOG.debug("[{}] Registered outstanding rpc {} ({}).", name(), id, type);
- rpcs.add(new OutstandingRpc(id, promise));
- }
-
- void discardRpc(long id) {
- LOG.debug("[{}] Discarding failed RPC {}.", name(), id);
- findRpc(id);
- }
-
- private static class OutstandingRpc {
- final long id;
- final Promise<Object> future;
-
- @SuppressWarnings("unchecked")
- OutstandingRpc(long id, Promise<?> future) {
- this.id = id;
- this.future = (Promise<Object>) future;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcException.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcException.java b/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcException.java
deleted file mode 100644
index d382df9..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcException.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.rpc;
-
-public class RpcException extends RuntimeException {
-
- RpcException(String remoteStackTrace) {
- super(remoteStackTrace);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcServer.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcServer.java b/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcServer.java
deleted file mode 100644
index 44db976..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcServer.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.rpc;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.BindException;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.SocketException;
-import java.security.SecureRandom;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import javax.security.auth.callback.Callback;
-import javax.security.auth.callback.CallbackHandler;
-import javax.security.auth.callback.NameCallback;
-import javax.security.auth.callback.PasswordCallback;
-import javax.security.sasl.AuthorizeCallback;
-import javax.security.sasl.RealmCallback;
-import javax.security.sasl.Sasl;
-import javax.security.sasl.SaslException;
-import javax.security.sasl.SaslServer;
-
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelOption;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.util.concurrent.ScheduledFuture;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.rsc.RSCConf;
-import com.cloudera.livy.rsc.Utils;
-import static com.cloudera.livy.rsc.RSCConf.Entry.*;
-
-/**
- * An RPC server. The server matches remote clients based on a secret that is generated on
- * the server - the secret needs to be given to the client through some other mechanism for
- * this to work.
- */
-public class RpcServer implements Closeable {
-
- private static final Logger LOG = LoggerFactory.getLogger(RpcServer.class);
- private static final SecureRandom RND = new SecureRandom();
-
- private final String address;
- private Channel channel;
- private final EventLoopGroup group;
- private final int port;
- private final ConcurrentMap<String, ClientInfo> pendingClients;
- private final RSCConf config;
- private final String portRange;
- private static enum PortRangeSchema{START_PORT, END_PORT, MAX};
- private final String PORT_DELIMITER = "~";
- /**
- * Creating RPC Server
- * @param lconf
- * @throws IOException
- * @throws InterruptedException
- */
- public RpcServer(RSCConf lconf) throws IOException, InterruptedException {
- this.config = lconf;
- this.portRange = config.get(LAUNCHER_PORT_RANGE);
- this.group = new NioEventLoopGroup(
- this.config.getInt(RPC_MAX_THREADS),
- Utils.newDaemonThreadFactory("RPC-Handler-%d"));
- int [] portData = getPortNumberAndRange();
- int startingPortNumber = portData[PortRangeSchema.START_PORT.ordinal()];
- int endPort = portData[PortRangeSchema.END_PORT.ordinal()];
- boolean isContected = false;
- for(int tries = startingPortNumber ; tries<=endPort ; tries++){
- try {
- this.channel = getChannel(tries);
- isContected = true;
- break;
- } catch(SocketException e){
- LOG.debug("RPC not able to connect port " + tries + " " + e.getMessage());
- }
- }
- if(!isContected) {
- throw new IOException("Unable to connect to provided ports " + this.portRange);
- }
- this.port = ((InetSocketAddress) channel.localAddress()).getPort();
- this.pendingClients = new ConcurrentHashMap<>();
- LOG.info("Connected to the port " + this.port);
- String address = config.get(RPC_SERVER_ADDRESS);
- if (address == null) {
- address = config.findLocalAddress();
- }
- this.address = address;
- }
-
- /**
- * Get Port Numbers
- */
- private int[] getPortNumberAndRange() throws ArrayIndexOutOfBoundsException,
- NumberFormatException {
- String[] split = this.portRange.split(PORT_DELIMITER);
- int [] portRange = new int [PortRangeSchema.MAX.ordinal()];
- try {
- portRange[PortRangeSchema.START_PORT.ordinal()] =
- Integer.parseInt(split[PortRangeSchema.START_PORT.ordinal()]);
- portRange[PortRangeSchema.END_PORT.ordinal()] =
- Integer.parseInt(split[PortRangeSchema.END_PORT.ordinal()]);
- } catch(ArrayIndexOutOfBoundsException e) {
- LOG.error("Port Range format is not correct " + this.portRange);
- throw e;
- } catch(NumberFormatException e) {
- LOG.error("Port are not in numeric format " + this.portRange);
- throw e;
- }
- return portRange;
- }
- /**
- * @throws InterruptedException
- **/
- private Channel getChannel(int portNumber) throws BindException, InterruptedException {
- Channel channel = new ServerBootstrap()
- .group(group)
- .channel(NioServerSocketChannel.class)
- .childHandler(new ChannelInitializer<SocketChannel>() {
- @Override
- public void initChannel(SocketChannel ch) throws Exception {
- SaslServerHandler saslHandler = new SaslServerHandler(config);
- final Rpc newRpc = Rpc.createServer(saslHandler, config, ch, group);
- saslHandler.rpc = newRpc;
-
- Runnable cancelTask = new Runnable() {
- @Override
- public void run() {
- LOG.warn("Timed out waiting for hello from client.");
- newRpc.close();
- }
- };
- saslHandler.cancelTask = group.schedule(cancelTask,
- config.getTimeAsMs(RPC_CLIENT_HANDSHAKE_TIMEOUT),
- TimeUnit.MILLISECONDS);
- }
- })
- .option(ChannelOption.SO_BACKLOG, 1)
- .option(ChannelOption.SO_REUSEADDR, true)
- .childOption(ChannelOption.SO_KEEPALIVE, true)
- .bind(portNumber)
- .sync()
- .channel();
- return channel;
- }
- /**
- * Tells the RPC server to expect connections from clients.
- *
- * @param clientId An identifier for the client. Must be unique.
- * @param secret The secret the client will send to the server to identify itself.
- * @param callback The callback for when a new client successfully connects with the given
- * credentials.
- */
- public void registerClient(String clientId, String secret, ClientCallback callback) {
- final ClientInfo client = new ClientInfo(clientId, secret, callback);
- if (pendingClients.putIfAbsent(clientId, client) != null) {
- throw new IllegalStateException(
- String.format("Client '%s' already registered.", clientId));
- }
- }
-
- /**
- * Stop waiting for connections for a given client ID.
- *
- * @param clientId The client ID to forget.
- */
- public void unregisterClient(String clientId) {
- pendingClients.remove(clientId);
- }
-
- /**
- * Creates a secret for identifying a client connection.
- */
- public String createSecret() {
- byte[] secret = new byte[config.getInt(RPC_SECRET_RANDOM_BITS) / 8];
- RND.nextBytes(secret);
-
- StringBuilder sb = new StringBuilder();
- for (byte b : secret) {
- if (b < 10) {
- sb.append("0");
- }
- sb.append(Integer.toHexString(b));
- }
- return sb.toString();
- }
-
- public String getAddress() {
- return address;
- }
-
- public int getPort() {
- return port;
- }
-
- public EventLoopGroup getEventLoopGroup() {
- return group;
- }
-
- @Override
- public void close() {
- try {
- channel.close();
- pendingClients.clear();
- } finally {
- group.shutdownGracefully();
- }
- }
-
- /**
- * A callback that can be registered to be notified when new clients are created and
- * successfully authenticate against the server.
- */
- public interface ClientCallback {
-
- /**
- * Called when a new client successfully connects.
- *
- * @param client The RPC instance for the new client.
- * @return The RpcDispatcher to be used for the client.
- */
- RpcDispatcher onNewClient(Rpc client);
-
-
- /**
- * Called when a new client successfully completed SASL authentication.
- *
- * @param client The RPC instance for the new client.
- */
- void onSaslComplete(Rpc client);
- }
-
- private class SaslServerHandler extends SaslHandler implements CallbackHandler {
-
- private final SaslServer server;
- private Rpc rpc;
- private ScheduledFuture<?> cancelTask;
- private String clientId;
- private ClientInfo client;
-
- SaslServerHandler(RSCConf config) throws IOException {
- super(config);
- this.server = Sasl.createSaslServer(config.get(SASL_MECHANISMS), Rpc.SASL_PROTOCOL,
- Rpc.SASL_REALM, config.getSaslOptions(), this);
- }
-
- @Override
- protected boolean isComplete() {
- return server.isComplete();
- }
-
- @Override
- protected String getNegotiatedProperty(String name) {
- return (String) server.getNegotiatedProperty(name);
- }
-
- @Override
- protected Rpc.SaslMessage update(Rpc.SaslMessage challenge) throws IOException {
- if (clientId == null) {
- Utils.checkArgument(challenge.clientId != null,
- "Missing client ID in SASL handshake.");
- clientId = challenge.clientId;
- client = pendingClients.get(clientId);
- Utils.checkArgument(client != null,
- "Unexpected client ID '%s' in SASL handshake.", clientId);
- }
-
- return new Rpc.SaslMessage(server.evaluateResponse(challenge.payload));
- }
-
- @Override
- public byte[] wrap(byte[] data, int offset, int len) throws IOException {
- return server.wrap(data, offset, len);
- }
-
- @Override
- public byte[] unwrap(byte[] data, int offset, int len) throws IOException {
- return server.unwrap(data, offset, len);
- }
-
- @Override
- public void dispose() throws IOException {
- if (!server.isComplete()) {
- onError(new SaslException("Server closed before SASL negotiation finished."));
- }
- server.dispose();
- }
-
- @Override
- protected void onComplete() throws Exception {
- cancelTask.cancel(true);
-
- RpcDispatcher dispatcher = null;
- try {
- dispatcher = client.callback.onNewClient(rpc);
- } catch (Exception e) {
- LOG.warn("Client callback threw an exception.", e);
- }
-
- if (dispatcher != null) {
- rpc.setDispatcher(dispatcher);
- }
-
- client.callback.onSaslComplete(rpc);
- }
-
- @Override
- protected void onError(Throwable error) {
- cancelTask.cancel(true);
- }
-
- @Override
- public void handle(Callback[] callbacks) {
- Utils.checkState(client != null, "Handshake not initialized yet.");
- for (Callback cb : callbacks) {
- if (cb instanceof NameCallback) {
- ((NameCallback)cb).setName(clientId);
- } else if (cb instanceof PasswordCallback) {
- ((PasswordCallback)cb).setPassword(client.secret.toCharArray());
- } else if (cb instanceof AuthorizeCallback) {
- ((AuthorizeCallback) cb).setAuthorized(true);
- } else if (cb instanceof RealmCallback) {
- RealmCallback rb = (RealmCallback) cb;
- rb.setText(rb.getDefaultText());
- }
- }
- }
-
- }
-
- private static class ClientInfo {
-
- final String id;
- final String secret;
- final ClientCallback callback;
-
- private ClientInfo(String id, String secret, ClientCallback callback) {
- this.id = id;
- this.secret = secret;
- this.callback = callback;
- }
-
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/main/java/com/cloudera/livy/rsc/rpc/SaslHandler.java
----------------------------------------------------------------------
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/SaslHandler.java b/rsc/src/main/java/com/cloudera/livy/rsc/rpc/SaslHandler.java
deleted file mode 100644
index 094fa1c..0000000
--- a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/SaslHandler.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.rpc;
-
-import java.io.IOException;
-import javax.security.sasl.Sasl;
-import javax.security.sasl.SaslException;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.livy.rsc.RSCConf;
-
-/**
- * Abstract SASL handler. Abstracts the auth protocol handling and encryption, if it's enabled.
- * Needs subclasses to provide access to the actual underlying SASL implementation (client or
- * server).
- */
-abstract class SaslHandler extends SimpleChannelInboundHandler<Rpc.SaslMessage>
- implements KryoMessageCodec.EncryptionHandler {
-
- // LOG is not static to make debugging easier (being able to identify which sub-class
- // generated the log message).
- private final Logger LOG;
- private final boolean requiresEncryption;
- private KryoMessageCodec kryo;
- private boolean hasAuthResponse = false;
-
- protected SaslHandler(RSCConf config) {
- this.requiresEncryption = Rpc.SASL_AUTH_CONF.equals(config.get(RSCConf.Entry.SASL_QOP));
- this.LOG = LoggerFactory.getLogger(getClass());
- }
-
- // Use a separate method to make it easier to create a SaslHandler without having to
- // plumb the KryoMessageCodec instance through the constructors.
- void setKryoMessageCodec(KryoMessageCodec kryo) {
- this.kryo = kryo;
- }
-
- @Override
- protected final void channelRead0(ChannelHandlerContext ctx, Rpc.SaslMessage msg)
- throws Exception {
- LOG.debug("Handling SASL challenge message...");
- Rpc.SaslMessage response = update(msg);
- if (response != null) {
- LOG.debug("Sending SASL challenge response...");
- hasAuthResponse = true;
- ctx.channel().writeAndFlush(response).sync();
- }
-
- if (!isComplete()) {
- return;
- }
-
- // If negotiation is complete, remove this handler from the pipeline, and register it with
- // the Kryo instance to handle encryption if needed.
- ctx.channel().pipeline().remove(this);
- String qop = getNegotiatedProperty(Sasl.QOP);
- LOG.debug("SASL negotiation finished with QOP {}.", qop);
- if (Rpc.SASL_AUTH_CONF.equals(qop)) {
- LOG.info("SASL confidentiality enabled.");
- kryo.setEncryptionHandler(this);
- } else {
- if (requiresEncryption) {
- throw new SaslException("Encryption required, but SASL negotiation did not set it up.");
- }
- dispose();
- }
-
- onComplete();
- }
-
- @Override
- public void channelInactive(ChannelHandlerContext ctx) throws Exception {
- dispose();
- super.channelInactive(ctx);
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
- if (!isComplete()) {
- LOG.info("Exception in SASL negotiation.", cause);
- onError(cause);
- ctx.close();
- }
- ctx.fireExceptionCaught(cause);
- }
-
- protected abstract boolean isComplete();
-
- protected abstract String getNegotiatedProperty(String name);
-
- protected abstract Rpc.SaslMessage update(Rpc.SaslMessage challenge) throws IOException;
-
- protected abstract void onComplete() throws Exception;
-
- protected abstract void onError(Throwable t);
-
-}
[04/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequestSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequestSpec.scala b/server/src/test/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequestSpec.scala
deleted file mode 100644
index 168d487..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequestSpec.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.interactive
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import org.scalatest.FunSpec
-
-import com.cloudera.livy.LivyBaseUnitTestSuite
-import com.cloudera.livy.sessions.{PySpark, SessionKindModule}
-
-class CreateInteractiveRequestSpec extends FunSpec with LivyBaseUnitTestSuite {
-
- private val mapper = new ObjectMapper()
- .registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
- .registerModule(new SessionKindModule())
-
- describe("CreateInteractiveRequest") {
-
- it("should have default values for fields after deserialization") {
- val json = """{ "kind" : "pyspark" }"""
- val req = mapper.readValue(json, classOf[CreateInteractiveRequest])
- assert(req.kind === PySpark())
- assert(req.proxyUser === None)
- assert(req.jars === List())
- assert(req.pyFiles === List())
- assert(req.files === List())
- assert(req.driverMemory === None)
- assert(req.driverCores === None)
- assert(req.executorMemory === None)
- assert(req.executorCores === None)
- assert(req.numExecutors === None)
- assert(req.archives === List())
- assert(req.queue === None)
- assert(req.name === None)
- assert(req.conf === Map())
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/interactive/InteractiveSessionServletSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/interactive/InteractiveSessionServletSpec.scala b/server/src/test/scala/com/cloudera/livy/server/interactive/InteractiveSessionServletSpec.scala
deleted file mode 100644
index c546718..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/interactive/InteractiveSessionServletSpec.scala
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.interactive
-
-import java.util.concurrent.atomic.AtomicInteger
-import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
-
-import scala.collection.JavaConverters._
-import scala.concurrent.Future
-
-import org.json4s.jackson.Json4sScalaModule
-import org.mockito.Matchers._
-import org.mockito.Mockito._
-import org.mockito.invocation.InvocationOnMock
-import org.mockito.stubbing.Answer
-import org.scalatest.Entry
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.{ExecuteRequest, LivyConf}
-import com.cloudera.livy.client.common.HttpMessages.SessionInfo
-import com.cloudera.livy.rsc.driver.{Statement, StatementState}
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions._
-import com.cloudera.livy.utils.AppInfo
-
-class InteractiveSessionServletSpec extends BaseInteractiveServletSpec {
-
- mapper.registerModule(new Json4sScalaModule())
-
- class MockInteractiveSessionServlet(
- sessionManager: InteractiveSessionManager,
- conf: LivyConf)
- extends InteractiveSessionServlet(sessionManager, mock[SessionStore], conf) {
-
- private var statements = IndexedSeq[Statement]()
-
- override protected def createSession(req: HttpServletRequest): InteractiveSession = {
- val statementCounter = new AtomicInteger()
-
- val session = mock[InteractiveSession]
- when(session.kind).thenReturn(Spark())
- when(session.appId).thenReturn(None)
- when(session.appInfo).thenReturn(AppInfo())
- when(session.logLines()).thenReturn(IndexedSeq())
- when(session.state).thenReturn(SessionState.Idle())
- when(session.stop()).thenReturn(Future.successful(()))
- when(session.proxyUser).thenReturn(None)
- when(session.statements).thenAnswer(
- new Answer[IndexedSeq[Statement]]() {
- override def answer(args: InvocationOnMock): IndexedSeq[Statement] = statements
- })
- when(session.executeStatement(any(classOf[ExecuteRequest]))).thenAnswer(
- new Answer[Statement]() {
- override def answer(args: InvocationOnMock): Statement = {
- val id = statementCounter.getAndIncrement
- val statement = new Statement(id, "1+1", StatementState.Available, "1")
-
- statements :+= statement
- statement
- }
- })
- when(session.cancelStatement(anyInt())).thenAnswer(
- new Answer[Unit] {
- override def answer(args: InvocationOnMock): Unit = {
- statements = IndexedSeq(
- new Statement(statementCounter.get(), null, StatementState.Cancelled, null))
- }
- }
- )
-
- session
- }
-
- }
-
- override def createServlet(): InteractiveSessionServlet = {
- val conf = createConf()
- val sessionManager = new InteractiveSessionManager(conf, mock[SessionStore], Some(Seq.empty))
- new MockInteractiveSessionServlet(sessionManager, conf)
- }
-
- it("should setup and tear down an interactive session") {
- jget[Map[String, Any]]("/") { data =>
- data("sessions") should equal(Seq())
- }
-
- jpost[Map[String, Any]]("/", createRequest()) { data =>
- header("Location") should equal("/0")
- data("id") should equal (0)
-
- val session = servlet.sessionManager.get(0)
- session should be (defined)
- }
-
- jget[Map[String, Any]]("/0") { data =>
- data("id") should equal (0)
- data("state") should equal ("idle")
-
- val batch = servlet.sessionManager.get(0)
- batch should be (defined)
- }
-
- jpost[Map[String, Any]]("/0/statements", ExecuteRequest("foo")) { data =>
- data("id") should be (0)
- data("code") shouldBe "1+1"
- data("progress") should be (0.0)
- data("output") shouldBe 1
- }
-
- jget[Map[String, Any]]("/0/statements") { data =>
- data("total_statements") should be (1)
- data("statements").asInstanceOf[Seq[Map[String, Any]]](0)("id") should be (0)
- }
-
- jpost[Map[String, Any]]("/0/statements/0/cancel", null, HttpServletResponse.SC_OK) { data =>
- data should equal(Map("msg" -> "canceled"))
- }
-
- jget[Map[String, Any]]("/0/statements") { data =>
- data("total_statements") should be (1)
- data("statements").asInstanceOf[Seq[Map[String, Any]]](0)("state") should be ("cancelled")
- }
-
- jdelete[Map[String, Any]]("/0") { data =>
- data should equal (Map("msg" -> "deleted"))
-
- val session = servlet.sessionManager.get(0)
- session should not be defined
- }
- }
-
- it("should show session properties") {
- val id = 0
- val appId = "appid"
- val owner = "owner"
- val proxyUser = "proxyUser"
- val state = SessionState.Running()
- val kind = Spark()
- val appInfo = AppInfo(Some("DRIVER LOG URL"), Some("SPARK UI URL"))
- val log = IndexedSeq[String]("log1", "log2")
-
- val session = mock[InteractiveSession]
- when(session.id).thenReturn(id)
- when(session.appId).thenReturn(Some(appId))
- when(session.owner).thenReturn(owner)
- when(session.proxyUser).thenReturn(Some(proxyUser))
- when(session.state).thenReturn(state)
- when(session.kind).thenReturn(kind)
- when(session.appInfo).thenReturn(appInfo)
- when(session.logLines()).thenReturn(log)
-
- val req = mock[HttpServletRequest]
-
- val view = servlet.asInstanceOf[InteractiveSessionServlet].clientSessionView(session, req)
- .asInstanceOf[SessionInfo]
-
- view.id shouldEqual id
- view.appId shouldEqual appId
- view.owner shouldEqual owner
- view.proxyUser shouldEqual proxyUser
- view.state shouldEqual state.toString
- view.kind shouldEqual kind.toString
- view.appInfo should contain (Entry(AppInfo.DRIVER_LOG_URL_NAME, appInfo.driverLogUrl.get))
- view.appInfo should contain (Entry(AppInfo.SPARK_UI_URL_NAME, appInfo.sparkUiUrl.get))
- view.log shouldEqual log.asJava
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/interactive/InteractiveSessionSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/interactive/InteractiveSessionSpec.scala b/server/src/test/scala/com/cloudera/livy/server/interactive/InteractiveSessionSpec.scala
deleted file mode 100644
index 7d92a43..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/interactive/InteractiveSessionSpec.scala
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.interactive
-
-import java.net.URI
-
-import scala.concurrent.Await
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import org.apache.spark.launcher.SparkLauncher
-import org.json4s.{DefaultFormats, Extraction, JValue}
-import org.json4s.jackson.JsonMethods.parse
-import org.mockito.{Matchers => MockitoMatchers}
-import org.mockito.Matchers._
-import org.mockito.Mockito.{atLeastOnce, verify, when}
-import org.scalatest.{BeforeAndAfterAll, FunSpec, Matchers}
-import org.scalatest.concurrent.Eventually._
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.{ExecuteRequest, JobHandle, LivyBaseUnitTestSuite, LivyConf}
-import com.cloudera.livy.rsc.{PingJob, RSCClient, RSCConf}
-import com.cloudera.livy.rsc.driver.StatementState
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.{PySpark, SessionState, Spark}
-import com.cloudera.livy.utils.{AppInfo, SparkApp}
-
-class InteractiveSessionSpec extends FunSpec
- with Matchers with BeforeAndAfterAll with LivyBaseUnitTestSuite {
-
- private val livyConf = new LivyConf()
- livyConf.set(LivyConf.REPL_JARS, "dummy.jar")
- .set(LivyConf.LIVY_SPARK_VERSION, "1.6.0")
- .set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.10.5")
-
- implicit val formats = DefaultFormats
-
- private var session: InteractiveSession = null
-
- private def createSession(
- sessionStore: SessionStore = mock[SessionStore],
- mockApp: Option[SparkApp] = None): InteractiveSession = {
- assume(sys.env.get("SPARK_HOME").isDefined, "SPARK_HOME is not set.")
-
- val req = new CreateInteractiveRequest()
- req.kind = PySpark()
- req.driverMemory = Some("512m")
- req.driverCores = Some(1)
- req.executorMemory = Some("512m")
- req.executorCores = Some(1)
- req.name = Some("InteractiveSessionSpec")
- req.conf = Map(
- SparkLauncher.DRIVER_EXTRA_CLASSPATH -> sys.props("java.class.path"),
- RSCConf.Entry.LIVY_JARS.key() -> ""
- )
- InteractiveSession.create(0, null, None, livyConf, req, sessionStore, mockApp)
- }
-
- private def executeStatement(code: String): JValue = {
- val id = session.executeStatement(ExecuteRequest(code)).id
- eventually(timeout(30 seconds), interval(100 millis)) {
- val s = session.getStatement(id).get
- s.state.get() shouldBe StatementState.Available
- parse(s.output)
- }
- }
-
- override def afterAll(): Unit = {
- if (session != null) {
- Await.ready(session.stop(), 30 seconds)
- session = null
- }
- super.afterAll()
- }
-
- private def withSession(desc: String)(fn: (InteractiveSession) => Unit): Unit = {
- it(desc) {
- assume(session != null, "No active session.")
- eventually(timeout(30 seconds), interval(100 millis)) {
- session.state shouldBe a[SessionState.Idle]
- }
- fn(session)
- }
- }
-
- describe("A spark session") {
-
- it("should get scala version matched jars with livy.repl.jars") {
- val testedJars = Seq(
- "test_2.10-0.1.jar",
- "local://dummy-path/test/test1_2.10-1.0.jar",
- "file:///dummy-path/test/test2_2.11-1.0-SNAPSHOT.jar",
- "hdfs:///dummy-path/test/test3.jar",
- "non-jar",
- "dummy.jar"
- )
- val livyConf = new LivyConf(false)
- .set(LivyConf.REPL_JARS, testedJars.mkString(","))
- .set(LivyConf.LIVY_SPARK_VERSION, "1.6.2")
- .set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.10")
- val properties = InteractiveSession.prepareBuilderProp(Map.empty, Spark(), livyConf)
- assert(properties(LivyConf.SPARK_JARS).split(",").toSet === Set("test_2.10-0.1.jar",
- "local://dummy-path/test/test1_2.10-1.0.jar",
- "hdfs:///dummy-path/test/test3.jar",
- "dummy.jar"))
-
- livyConf.set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.11")
- val properties1 = InteractiveSession.prepareBuilderProp(Map.empty, Spark(), livyConf)
- assert(properties1(LivyConf.SPARK_JARS).split(",").toSet === Set(
- "file:///dummy-path/test/test2_2.11-1.0-SNAPSHOT.jar",
- "hdfs:///dummy-path/test/test3.jar",
- "dummy.jar"))
- }
-
-
- it("should set rsc jars through livy conf") {
- val rscJars = Set(
- "dummy.jar",
- "local:///dummy-path/dummy1.jar",
- "file:///dummy-path/dummy2.jar",
- "hdfs:///dummy-path/dummy3.jar")
- val livyConf = new LivyConf(false)
- .set(LivyConf.REPL_JARS, "dummy.jar")
- .set(LivyConf.RSC_JARS, rscJars.mkString(","))
- .set(LivyConf.LIVY_SPARK_VERSION, "1.6.2")
- .set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.10")
- val properties = InteractiveSession.prepareBuilderProp(Map.empty, Spark(), livyConf)
- // if livy.rsc.jars is configured in LivyConf, it should be passed to RSCConf.
- properties(RSCConf.Entry.LIVY_JARS.key()).split(",").toSet === rscJars
-
- val rscJars1 = Set(
- "foo.jar",
- "local:///dummy-path/foo1.jar",
- "file:///dummy-path/foo2.jar",
- "hdfs:///dummy-path/foo3.jar")
- val properties1 = InteractiveSession.prepareBuilderProp(
- Map(RSCConf.Entry.LIVY_JARS.key() -> rscJars1.mkString(",")), Spark(), livyConf)
- // if rsc jars are configured both in LivyConf and RSCConf, RSCConf should take precedence.
- properties1(RSCConf.Entry.LIVY_JARS.key()).split(",").toSet === rscJars1
- }
-
- it("should start in the idle state") {
- session = createSession()
- session.state should (be(a[SessionState.Starting]) or be(a[SessionState.Idle]))
- }
-
- it("should update appId and appInfo and session store") {
- val mockApp = mock[SparkApp]
- val sessionStore = mock[SessionStore]
- val session = createSession(sessionStore, Some(mockApp))
-
- val expectedAppId = "APPID"
- session.appIdKnown(expectedAppId)
- session.appId shouldEqual Some(expectedAppId)
-
- val expectedAppInfo = AppInfo(Some("DRIVER LOG URL"), Some("SPARK UI URL"))
- session.infoChanged(expectedAppInfo)
- session.appInfo shouldEqual expectedAppInfo
-
- verify(sessionStore, atLeastOnce()).save(
- MockitoMatchers.eq(InteractiveSession.RECOVERY_SESSION_TYPE), anyObject())
- }
-
- withSession("should execute `1 + 2` == 3") { session =>
- val result = executeStatement("1 + 2")
- val expectedResult = Extraction.decompose(Map(
- "status" -> "ok",
- "execution_count" -> 0,
- "data" -> Map(
- "text/plain" -> "3"
- )
- ))
-
- result should equal (expectedResult)
- }
-
- withSession("should report an error if accessing an unknown variable") { session =>
- val result = executeStatement("x")
- val expectedResult = Extraction.decompose(Map(
- "status" -> "error",
- "execution_count" -> 1,
- "ename" -> "NameError",
- "evalue" -> "name 'x' is not defined",
- "traceback" -> List(
- "Traceback (most recent call last):\n",
- "NameError: name 'x' is not defined\n"
- )
- ))
-
- result should equal (expectedResult)
- eventually(timeout(10 seconds), interval(30 millis)) {
- session.state shouldBe a[SessionState.Idle]
- }
- }
-
- withSession("should get statement progress along with statement result") { session =>
- val code =
- """
- |from time import sleep
- |sleep(3)
- """.stripMargin
- val statement = session.executeStatement(ExecuteRequest(code))
- statement.progress should be (0.0)
-
- eventually(timeout(10 seconds), interval(100 millis)) {
- val s = session.getStatement(statement.id).get
- s.state.get() shouldBe StatementState.Available
- s.progress should be (1.0)
- }
- }
-
- withSession("should error out the session if the interpreter dies") { session =>
- session.executeStatement(ExecuteRequest("import os; os._exit(666)"))
- eventually(timeout(30 seconds), interval(100 millis)) {
- session.state shouldBe a[SessionState.Error]
- }
- }
- }
-
- describe("recovery") {
- it("should recover session") {
- val conf = new LivyConf()
- val sessionStore = mock[SessionStore]
- val mockClient = mock[RSCClient]
- when(mockClient.submit(any(classOf[PingJob]))).thenReturn(mock[JobHandle[Void]])
- val m =
- InteractiveRecoveryMetadata(
- 78, None, "appTag", Spark(), 0, null, None, Some(URI.create("")))
- val s = InteractiveSession.recover(m, conf, sessionStore, None, Some(mockClient))
-
- s.state shouldBe a[SessionState.Recovering]
-
- s.appIdKnown("appId")
- verify(sessionStore, atLeastOnce()).save(
- MockitoMatchers.eq(InteractiveSession.RECOVERY_SESSION_TYPE), anyObject())
- }
-
- it("should recover session to dead state if rscDriverUri is unknown") {
- val conf = new LivyConf()
- val sessionStore = mock[SessionStore]
- val m = InteractiveRecoveryMetadata(
- 78, Some("appId"), "appTag", Spark(), 0, null, None, None)
- val s = InteractiveSession.recover(m, conf, sessionStore, None)
-
- s.state shouldBe a[SessionState.Dead]
- s.logLines().mkString should include("RSCDriver URI is unknown")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/interactive/JobApiSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/interactive/JobApiSpec.scala b/server/src/test/scala/com/cloudera/livy/server/interactive/JobApiSpec.scala
deleted file mode 100644
index f8c7c3f..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/interactive/JobApiSpec.scala
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.interactive
-
-import java.io.File
-import java.net.URI
-import java.nio.ByteBuffer
-import java.nio.file.{Files, Paths}
-import javax.servlet.http.HttpServletResponse._
-
-import scala.concurrent.duration._
-import scala.io.Source
-import scala.language.postfixOps
-
-import org.scalatest.concurrent.Eventually._
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.{Job, JobHandle}
-import com.cloudera.livy.client.common.{BufferUtils, Serializer}
-import com.cloudera.livy.client.common.HttpMessages._
-import com.cloudera.livy.server.RemoteUserOverride
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.{InteractiveSessionManager, SessionState}
-import com.cloudera.livy.test.jobs.{Echo, GetCurrentUser}
-
-class JobApiSpec extends BaseInteractiveServletSpec {
-
- private val PROXY = "__proxy__"
-
- private var sessionId: Int = -1
-
- override def createServlet(): InteractiveSessionServlet = {
- val conf = createConf()
- val sessionStore = mock[SessionStore]
- val sessionManager = new InteractiveSessionManager(conf, sessionStore, Some(Seq.empty))
- new InteractiveSessionServlet(sessionManager, sessionStore, conf) with RemoteUserOverride
- }
-
- def withSessionId(desc: String)(fn: (Int) => Unit): Unit = {
- it(desc) {
- assume(sessionId != -1, "No active session.")
- fn(sessionId)
- }
- }
-
- describe("Interactive Servlet") {
-
- it("should create sessions") {
- jpost[SessionInfo]("/", createRequest()) { data =>
- waitForIdle(data.id)
- header("Location") should equal("/0")
- data.id should equal (0)
- sessionId = data.id
- }
- }
-
- withSessionId("should handle asynchronous jobs") { testJobSubmission(_, false) }
-
- withSessionId("should handle synchronous jobs") { testJobSubmission(_, true) }
-
- // Test that the file does get copied over to the live home dir on HDFS - does not test end
- // to end that the RSCClient class copies it over to the app.
- withSessionId("should support file uploads") { id =>
- testResourceUpload("file", id)
- }
-
- withSessionId("should support jar uploads") { id =>
- testResourceUpload("jar", id)
- }
-
- withSessionId("should monitor async Spark jobs") { sid =>
- val ser = new Serializer()
- val job = BufferUtils.toByteArray(ser.serialize(new Echo("hello")))
- var jobId: Long = -1L
- jpost[JobStatus](s"/$sid/submit-job", new SerializedJob(job)) { status =>
- jobId = status.id
- }
-
- eventually(timeout(1 minute), interval(100 millis)) {
- jget[JobStatus](s"/$sid/jobs/$jobId") { status =>
- status.state should be (JobHandle.State.SUCCEEDED)
- }
- }
- }
-
- withSessionId("should update last activity on connect") { sid =>
- val currentActivity = servlet.sessionManager.get(sid).get.lastActivity
- jpost[SessionInfo](s"/$sid/connect", null, expectedStatus = SC_OK) { info =>
- val newActivity = servlet.sessionManager.get(sid).get.lastActivity
- assert(newActivity > currentActivity)
- }
- }
-
- withSessionId("should tear down sessions") { id =>
- jdelete[Map[String, Any]](s"/$id") { data =>
- data should equal (Map("msg" -> "deleted"))
- }
- jget[Map[String, Any]]("/") { data =>
- data("sessions") match {
- case contents: Seq[_] => contents.size should equal (0)
- case _ => fail("Response is not an array.")
- }
- }
-
- // Make sure the session's staging directory was cleaned up.
- assert(tempDir.listFiles().length === 0)
- }
-
- it("should support user impersonation") {
- val headers = makeUserHeaders(PROXY)
- jpost[SessionInfo]("/", createRequest(inProcess = false), headers = headers) { data =>
- try {
- waitForIdle(data.id)
- data.owner should be (PROXY)
- data.proxyUser should be (PROXY)
- val user = runJob(data.id, new GetCurrentUser(), headers = headers)
- user should be (PROXY)
- } finally {
- deleteSession(data.id)
- }
- }
- }
-
- it("should honor impersonation requests") {
- val request = createRequest(inProcess = false)
- request.proxyUser = Some(PROXY)
- jpost[SessionInfo]("/", request, headers = adminHeaders) { data =>
- try {
- waitForIdle(data.id)
- data.owner should be (ADMIN)
- data.proxyUser should be (PROXY)
- val user = runJob(data.id, new GetCurrentUser(), headers = adminHeaders)
- user should be (PROXY)
-
- // Test that files are uploaded to a new session directory.
- assert(tempDir.listFiles().length === 0)
- testResourceUpload("file", data.id)
- } finally {
- deleteSession(data.id)
- assert(tempDir.listFiles().length === 0)
- }
- }
- }
-
- it("should respect config black list") {
- jpost[SessionInfo]("/", createRequest(extraConf = BLACKLISTED_CONFIG),
- expectedStatus = SC_BAD_REQUEST) { _ => }
- }
-
- }
-
- private def waitForIdle(id: Int): Unit = {
- eventually(timeout(1 minute), interval(100 millis)) {
- jget[SessionInfo](s"/$id") { status =>
- status.state should be (SessionState.Idle().toString())
- }
- }
- }
-
- private def deleteSession(id: Int): Unit = {
- jdelete[Map[String, Any]](s"/$id", headers = adminHeaders) { _ => }
- }
-
- private def testResourceUpload(cmd: String, sessionId: Int): Unit = {
- val f = File.createTempFile("uploadTestFile", cmd)
- val conf = createConf()
-
- Files.write(Paths.get(f.getAbsolutePath), "Test data".getBytes())
-
- jupload[Unit](s"/$sessionId/upload-$cmd", Map(cmd -> f), expectedStatus = SC_OK) { _ =>
- // There should be a single directory under the staging dir.
- val subdirs = tempDir.listFiles()
- assert(subdirs.length === 1)
- val stagingDir = subdirs(0).toURI().toString()
-
- val resultFile = new File(new URI(s"$stagingDir/${f.getName}"))
- resultFile.deleteOnExit()
- resultFile.exists() should be(true)
- Source.fromFile(resultFile).mkString should be("Test data")
- }
- }
-
- private def testJobSubmission(sid: Int, sync: Boolean): Unit = {
- val result = runJob(sid, new Echo(42), sync = sync)
- result should be (42)
- }
-
- private def runJob[T](
- sid: Int,
- job: Job[T],
- sync: Boolean = false,
- headers: Map[String, String] = defaultHeaders): T = {
- val ser = new Serializer()
- val jobData = BufferUtils.toByteArray(ser.serialize(job))
- val route = if (sync) s"/$sid/submit-job" else s"/$sid/run-job"
- var jobId: Long = -1L
- jpost[JobStatus](route, new SerializedJob(jobData), headers = headers) { data =>
- jobId = data.id
- }
-
- var result: Option[T] = None
- eventually(timeout(1 minute), interval(100 millis)) {
- jget[JobStatus](s"/$sid/jobs/$jobId") { status =>
- status.id should be (jobId)
- status.state should be (JobHandle.State.SUCCEEDED)
- result = Some(ser.deserialize(ByteBuffer.wrap(status.result)).asInstanceOf[T])
- }
- }
- result.getOrElse(throw new IllegalStateException())
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/interactive/SessionHeartbeatSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/interactive/SessionHeartbeatSpec.scala b/server/src/test/scala/com/cloudera/livy/server/interactive/SessionHeartbeatSpec.scala
deleted file mode 100644
index 36eb7ef..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/interactive/SessionHeartbeatSpec.scala
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.interactive
-
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import org.mockito.Mockito.{never, verify, when}
-import org.scalatest.{FunSpec, Matchers}
-import org.scalatest.concurrent.Eventually._
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.LivyConf
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.{Session, SessionManager}
-import com.cloudera.livy.sessions.Session.RecoveryMetadata
-
-class SessionHeartbeatSpec extends FunSpec with Matchers {
- describe("SessionHeartbeat") {
- class TestHeartbeat(override val heartbeatTimeout: FiniteDuration) extends SessionHeartbeat {}
-
- it("should not expire if heartbeat was never called.") {
- val t = new TestHeartbeat(Duration.Zero)
- t.heartbeatExpired shouldBe false
- }
-
- it("should expire if time has elapsed.") {
- val t = new TestHeartbeat(Duration.fromNanos(1))
- t.heartbeat()
- eventually(timeout(2 nano), interval(1 nano)) {
- t.heartbeatExpired shouldBe true
- }
- }
-
- it("should not expire if time hasn't elapsed.") {
- val t = new TestHeartbeat(Duration.create(1, DAYS))
- t.heartbeat()
- t.heartbeatExpired shouldBe false
- }
- }
-
- describe("SessionHeartbeatWatchdog") {
- abstract class TestSession extends Session(0, null, null) with SessionHeartbeat {}
- class TestWatchdog(conf: LivyConf)
- extends SessionManager[TestSession, RecoveryMetadata](
- conf,
- { _ => assert(false).asInstanceOf[TestSession] },
- mock[SessionStore],
- "test",
- Some(Seq.empty))
- with SessionHeartbeatWatchdog[TestSession, RecoveryMetadata] {}
-
- it("should delete only expired sessions") {
- val expiredSession: TestSession = mock[TestSession]
- when(expiredSession.id).thenReturn(0)
- when(expiredSession.heartbeatExpired).thenReturn(true)
-
- val nonExpiredSession: TestSession = mock[TestSession]
- when(nonExpiredSession.id).thenReturn(1)
- when(nonExpiredSession.heartbeatExpired).thenReturn(false)
-
- val n = new TestWatchdog(new LivyConf())
-
- n.register(expiredSession)
- n.register(nonExpiredSession)
- n.deleteExpiredSessions()
-
- verify(expiredSession).stop()
- verify(nonExpiredSession, never).stop()
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/recovery/BlackholeStateStoreSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/recovery/BlackholeStateStoreSpec.scala b/server/src/test/scala/com/cloudera/livy/server/recovery/BlackholeStateStoreSpec.scala
deleted file mode 100644
index c11feff..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/recovery/BlackholeStateStoreSpec.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.recovery
-
-import org.scalatest.FunSpec
-import org.scalatest.Matchers._
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf}
-
-class BlackholeStateStoreSpec extends FunSpec with LivyBaseUnitTestSuite {
- describe("BlackholeStateStore") {
- val stateStore = new BlackholeStateStore(new LivyConf())
-
- it("set should not throw") {
- stateStore.set("", 1.asInstanceOf[Object])
- }
-
- it("get should return None") {
- val v = stateStore.get[Object]("")
- v shouldBe None
- }
-
- it("getChildren should return empty list") {
- val c = stateStore.getChildren("")
- c shouldBe empty
- }
-
- it("remove should not throw") {
- stateStore.remove("")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/recovery/FileSystemStateStoreSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/recovery/FileSystemStateStoreSpec.scala b/server/src/test/scala/com/cloudera/livy/server/recovery/FileSystemStateStoreSpec.scala
deleted file mode 100644
index 9b7b0f3..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/recovery/FileSystemStateStoreSpec.scala
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.recovery
-
-import java.io.{FileNotFoundException, InputStream, IOException}
-import java.util
-
-import org.apache.hadoop.fs._
-import org.apache.hadoop.fs.Options.{CreateOpts, Rename}
-import org.apache.hadoop.fs.permission.FsPermission
-import org.hamcrest.Description
-import org.mockito.ArgumentMatcher
-import org.mockito.Matchers.{any, anyInt, argThat, eq => equal}
-import org.mockito.Mockito.{atLeastOnce, verify, when}
-import org.mockito.internal.matchers.Equals
-import org.mockito.invocation.InvocationOnMock
-import org.mockito.stubbing.Answer
-import org.scalatest.FunSpec
-import org.scalatest.Matchers._
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf}
-
-class FileSystemStateStoreSpec extends FunSpec with LivyBaseUnitTestSuite {
- describe("FileSystemStateStore") {
- def pathEq(wantedPath: String): Path = argThat(new ArgumentMatcher[Path] {
- private val matcher = new Equals(wantedPath)
-
- override def matches(path: Any): Boolean = matcher.matches(path.toString)
-
- override def describeTo(d: Description): Unit = { matcher.describeTo(d) }
- })
-
- def makeConf(): LivyConf = {
- val conf = new LivyConf()
- conf.set(LivyConf.RECOVERY_STATE_STORE_URL, "file://tmp/")
-
- conf
- }
-
- def mockFileContext(rootDirPermission: String): FileContext = {
- val fileContext = mock[FileContext]
- val rootDirStatus = mock[FileStatus]
- when(fileContext.getFileStatus(any())).thenReturn(rootDirStatus)
- when(rootDirStatus.getPermission).thenReturn(new FsPermission(rootDirPermission))
-
- fileContext
- }
-
- it("should throw if url is not configured") {
- intercept[IllegalArgumentException](new FileSystemStateStore(new LivyConf()))
- }
-
- it("should set and verify file permission") {
- val fileContext = mockFileContext("700")
- new FileSystemStateStore(makeConf(), Some(fileContext))
-
- verify(fileContext).setUMask(new FsPermission("077"))
- }
-
- it("should reject insecure permission") {
- def test(permission: String): Unit = {
- val fileContext = mockFileContext(permission)
-
- intercept[IllegalArgumentException](new FileSystemStateStore(makeConf(), Some(fileContext)))
- }
- test("600")
- test("400")
- test("677")
- test("670")
- test("607")
- }
-
- it("set should write with an intermediate file") {
- val fileContext = mockFileContext("700")
- val outputStream = mock[FSDataOutputStream]
- when(fileContext.create(pathEq("/key.tmp"), any[util.EnumSet[CreateFlag]], any[CreateOpts]))
- .thenReturn(outputStream)
-
- val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
-
- stateStore.set("key", "value")
-
- verify(outputStream).write(""""value"""".getBytes)
- verify(outputStream, atLeastOnce).close()
-
-
- verify(fileContext).rename(pathEq("/key.tmp"), pathEq("/key"), equal(Rename.OVERWRITE))
- verify(fileContext).delete(pathEq("/.key.tmp.crc"), equal(false))
- }
-
- it("get should read file") {
- val fileContext = mockFileContext("700")
- abstract class MockInputStream extends InputStream with Seekable with PositionedReadable {}
- val inputStream: InputStream = mock[MockInputStream]
- when(inputStream.read(any[Array[Byte]](), anyInt(), anyInt())).thenAnswer(new Answer[Int] {
- private var firstCall = true
- override def answer(invocation: InvocationOnMock): Int = {
- if (firstCall) {
- firstCall = false
- val buf = invocation.getArguments()(0).asInstanceOf[Array[Byte]]
- val b = """"value"""".getBytes()
- b.copyToArray(buf)
- b.length
- } else {
- -1
- }
- }
- })
-
- when(fileContext.open(pathEq("/key"))).thenReturn(new FSDataInputStream(inputStream))
-
- val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
-
- stateStore.get[String]("key") shouldBe Some("value")
-
- verify(inputStream, atLeastOnce).close()
- }
-
- it("get non-existent key should return None") {
- val fileContext = mockFileContext("700")
- when(fileContext.open(any())).thenThrow(new FileNotFoundException("Unit test"))
-
- val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
-
- stateStore.get[String]("key") shouldBe None
- }
-
- it("getChildren should list file") {
- val parentPath = "path"
- def makeFileStatus(name: String): FileStatus = {
- val fs = new FileStatus()
- fs.setPath(new Path(parentPath, name))
- fs
- }
- val children = Seq("c1", "c2")
-
- val fileContext = mockFileContext("700")
- val util = mock[FileContext#Util]
- when(util.listStatus(pathEq(s"/$parentPath")))
- .thenReturn(children.map(makeFileStatus).toArray)
- when(fileContext.util()).thenReturn(util)
-
- val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
- stateStore.getChildren(parentPath) should contain theSameElementsAs children
- }
-
- def getChildrenErrorTest(error: Exception): Unit = {
- val parentPath = "path"
-
- val fileContext = mockFileContext("700")
- val util = mock[FileContext#Util]
- when(util.listStatus(pathEq(s"/$parentPath"))).thenThrow(error)
- when(fileContext.util()).thenReturn(util)
-
- val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
- stateStore.getChildren(parentPath) shouldBe empty
- }
-
- it("getChildren should return empty list if the key doesn't exist") {
- getChildrenErrorTest(new IOException("Unit test"))
- }
-
- it("getChildren should return empty list if key doesn't exist") {
- getChildrenErrorTest(new FileNotFoundException("Unit test"))
- }
-
- it("remove should delete file") {
- val fileContext = mockFileContext("700")
-
- val stateStore = new FileSystemStateStore(makeConf(), Some(fileContext))
- stateStore.remove("key")
-
- verify(fileContext).delete(pathEq("/key"), equal(false))
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/recovery/SessionStoreSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/recovery/SessionStoreSpec.scala b/server/src/test/scala/com/cloudera/livy/server/recovery/SessionStoreSpec.scala
deleted file mode 100644
index 3435c2e..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/recovery/SessionStoreSpec.scala
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.recovery
-
-import scala.util.Success
-
-import org.mockito.Mockito._
-import org.scalatest.FunSpec
-import org.scalatest.Matchers._
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf}
-import com.cloudera.livy.sessions.Session.RecoveryMetadata
-
-class SessionStoreSpec extends FunSpec with LivyBaseUnitTestSuite {
- describe("SessionStore") {
- case class TestRecoveryMetadata(id: Int) extends RecoveryMetadata
-
- val sessionType = "test"
- val sessionPath = s"v1/$sessionType"
- val sessionManagerPath = s"v1/$sessionType/state"
-
- val conf = new LivyConf()
- it("should set session state and session counter when saving a session.") {
- val stateStore = mock[StateStore]
- val sessionStore = new SessionStore(conf, stateStore)
-
- val m = TestRecoveryMetadata(99)
- sessionStore.save(sessionType, m)
- verify(stateStore).set(s"$sessionPath/99", m)
- }
-
- it("should return existing sessions") {
- val validMetadata = Map(
- "0" -> Some(TestRecoveryMetadata(0)),
- "5" -> None,
- "77" -> Some(TestRecoveryMetadata(77)))
- val corruptedMetadata = Map(
- "7" -> new RuntimeException("Test"),
- "11212" -> new RuntimeException("Test")
- )
- val stateStore = mock[StateStore]
- val sessionStore = new SessionStore(conf, stateStore)
- when(stateStore.getChildren(sessionPath))
- .thenReturn((validMetadata ++ corruptedMetadata).keys.toList)
-
- validMetadata.foreach { case (id, m) =>
- when(stateStore.get[TestRecoveryMetadata](s"$sessionPath/$id")).thenReturn(m)
- }
-
- corruptedMetadata.foreach { case (id, ex) =>
- when(stateStore.get[TestRecoveryMetadata](s"$sessionPath/$id")).thenThrow(ex)
- }
-
- val s = sessionStore.getAllSessions[TestRecoveryMetadata](sessionType)
- // Verify normal metadata are retrieved.
- s.filter(_.isSuccess) should contain theSameElementsAs
- validMetadata.values.filter(_.isDefined).map(m => Success(m.get))
- // Verify exceptions are wrapped as in Try and are returned.
- s.filter(_.isFailure) should have size corruptedMetadata.size
- }
-
- it("should not throw if the state store is empty") {
- val stateStore = mock[StateStore]
- val sessionStore = new SessionStore(conf, stateStore)
- when(stateStore.getChildren(sessionPath)).thenReturn(Seq.empty)
-
- val s = sessionStore.getAllSessions[TestRecoveryMetadata](sessionType)
- s.filter(_.isSuccess) shouldBe empty
- }
-
- it("should return correct next session id") {
- val stateStore = mock[StateStore]
- val sessionStore = new SessionStore(conf, stateStore)
-
- when(stateStore.get[SessionManagerState](sessionManagerPath)).thenReturn(None)
- sessionStore.getNextSessionId(sessionType) shouldBe 0
-
- val sms = SessionManagerState(100)
- when(stateStore.get[SessionManagerState](sessionManagerPath)).thenReturn(Some(sms))
- sessionStore.getNextSessionId(sessionType) shouldBe sms.nextSessionId
- }
-
- it("should remove session") {
- val stateStore = mock[StateStore]
- val sessionStore = new SessionStore(conf, stateStore)
- val id = 1
-
- sessionStore.remove(sessionType, 1)
- verify(stateStore).remove(s"$sessionPath/$id")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/recovery/StateStoreSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/recovery/StateStoreSpec.scala b/server/src/test/scala/com/cloudera/livy/server/recovery/StateStoreSpec.scala
deleted file mode 100644
index c0c4918..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/recovery/StateStoreSpec.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.recovery
-
-import scala.reflect.classTag
-
-import org.scalatest.{BeforeAndAfter, FunSpec}
-import org.scalatest.Matchers._
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf}
-import com.cloudera.livy.sessions.SessionManager
-
-class StateStoreSpec extends FunSpec with BeforeAndAfter with LivyBaseUnitTestSuite {
- describe("StateStore") {
- after {
- StateStore.cleanup()
- }
-
- def createConf(stateStore: String): LivyConf = {
- val conf = new LivyConf()
- conf.set(LivyConf.RECOVERY_MODE.key, SessionManager.SESSION_RECOVERY_MODE_RECOVERY)
- conf.set(LivyConf.RECOVERY_STATE_STORE.key, stateStore)
- conf
- }
-
- it("should throw an error on get if it's not initialized") {
- intercept[AssertionError] { StateStore.get }
- }
-
- it("should initialize blackhole state store if recovery is disabled") {
- StateStore.init(new LivyConf())
- StateStore.get shouldBe a[BlackholeStateStore]
- }
-
- it("should pick the correct store according to state store config") {
- StateStore.pickStateStore(createConf("filesystem")) shouldBe classOf[FileSystemStateStore]
- StateStore.pickStateStore(createConf("zookeeper")) shouldBe classOf[ZooKeeperStateStore]
- }
-
- it("should return error if an unknown recovery mode is set") {
- val conf = new LivyConf()
- conf.set(LivyConf.RECOVERY_MODE.key, "unknown")
- intercept[IllegalArgumentException] { StateStore.init(conf) }
- }
-
- it("should return error if an unknown state store is set") {
- intercept[IllegalArgumentException] { StateStore.init(createConf("unknown")) }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/server/recovery/ZooKeeperStateStoreSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/server/recovery/ZooKeeperStateStoreSpec.scala b/server/src/test/scala/com/cloudera/livy/server/recovery/ZooKeeperStateStoreSpec.scala
deleted file mode 100644
index 860568f..0000000
--- a/server/src/test/scala/com/cloudera/livy/server/recovery/ZooKeeperStateStoreSpec.scala
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.server.recovery
-
-import scala.collection.JavaConverters._
-
-import org.apache.curator.framework.CuratorFramework
-import org.apache.curator.framework.api._
-import org.apache.curator.framework.listen.Listenable
-import org.apache.zookeeper.data.Stat
-import org.mockito.Mockito._
-import org.scalatest.FunSpec
-import org.scalatest.Matchers._
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf}
-
-class ZooKeeperStateStoreSpec extends FunSpec with LivyBaseUnitTestSuite {
- describe("ZooKeeperStateStore") {
- case class TestFixture(stateStore: ZooKeeperStateStore, curatorClient: CuratorFramework)
- val conf = new LivyConf()
- conf.set(LivyConf.RECOVERY_STATE_STORE_URL, "host")
- val key = "key"
- val prefixedKey = s"/livy/$key"
-
- def withMock[R](testBody: TestFixture => R): R = {
- val curatorClient = mock[CuratorFramework]
- when(curatorClient.getUnhandledErrorListenable())
- .thenReturn(mock[Listenable[UnhandledErrorListener]])
- val stateStore = new ZooKeeperStateStore(conf, Some(curatorClient))
- testBody(TestFixture(stateStore, curatorClient))
- }
-
- def mockExistsBuilder(curatorClient: CuratorFramework, exists: Boolean): Unit = {
- val existsBuilder = mock[ExistsBuilder]
- when(curatorClient.checkExists()).thenReturn(existsBuilder)
- if (exists) {
- when(existsBuilder.forPath(prefixedKey)).thenReturn(mock[Stat])
- }
- }
-
- it("should throw on bad config") {
- withMock { f =>
- val conf = new LivyConf()
- intercept[IllegalArgumentException] { new ZooKeeperStateStore(conf) }
-
- conf.set(LivyConf.RECOVERY_STATE_STORE_URL, "host")
- conf.set(ZooKeeperStateStore.ZK_RETRY_CONF, "bad")
- intercept[IllegalArgumentException] { new ZooKeeperStateStore(conf) }
- }
- }
-
- it("set should use curatorClient") {
- withMock { f =>
- mockExistsBuilder(f.curatorClient, true)
-
- val setDataBuilder = mock[SetDataBuilder]
- when(f.curatorClient.setData()).thenReturn(setDataBuilder)
-
- f.stateStore.set("key", 1.asInstanceOf[Object])
-
- verify(f.curatorClient).start()
- verify(setDataBuilder).forPath(prefixedKey, Array[Byte](49))
- }
- }
-
- it("set should create parents if they don't exist") {
- withMock { f =>
- mockExistsBuilder(f.curatorClient, false)
-
- val createBuilder = mock[CreateBuilder]
- when(f.curatorClient.create()).thenReturn(createBuilder)
- val p = mock[ProtectACLCreateModePathAndBytesable[String]]
- when(createBuilder.creatingParentsIfNeeded()).thenReturn(p)
-
- f.stateStore.set("key", 1.asInstanceOf[Object])
-
- verify(f.curatorClient).start()
- verify(p).forPath(prefixedKey, Array[Byte](49))
- }
- }
-
- it("get should retrieve retry policy configs") {
- conf.set(com.cloudera.livy.server.recovery.ZooKeeperStateStore.ZK_RETRY_CONF, "11,77")
- withMock { f =>
- mockExistsBuilder(f.curatorClient, true)
-
- f.stateStore.retryPolicy should not be null
- f.stateStore.retryPolicy.getN shouldBe 11
- }
- }
-
- it("get should retrieve data from curatorClient") {
- withMock { f =>
- mockExistsBuilder(f.curatorClient, true)
-
- val getDataBuilder = mock[GetDataBuilder]
- when(f.curatorClient.getData()).thenReturn(getDataBuilder)
- when(getDataBuilder.forPath(prefixedKey)).thenReturn(Array[Byte](50))
-
- val v = f.stateStore.get[Int]("key")
-
- verify(f.curatorClient).start()
- v shouldBe Some(2)
- }
- }
-
- it("get should return None if key doesn't exist") {
- withMock { f =>
- mockExistsBuilder(f.curatorClient, false)
-
- val v = f.stateStore.get[Int]("key")
-
- verify(f.curatorClient).start()
- v shouldBe None
- }
- }
-
- it("getChildren should use curatorClient") {
- withMock { f =>
- mockExistsBuilder(f.curatorClient, true)
-
- val getChildrenBuilder = mock[GetChildrenBuilder]
- when(f.curatorClient.getChildren()).thenReturn(getChildrenBuilder)
- val children = List("abc", "def")
- when(getChildrenBuilder.forPath(prefixedKey)).thenReturn(children.asJava)
-
- val c = f.stateStore.getChildren("key")
-
- verify(f.curatorClient).start()
- c shouldBe children
- }
- }
-
- it("getChildren should return empty list if key doesn't exist") {
- withMock { f =>
- mockExistsBuilder(f.curatorClient, false)
-
- val c = f.stateStore.getChildren("key")
-
- verify(f.curatorClient).start()
- c shouldBe empty
- }
- }
-
- it("remove should use curatorClient") {
- withMock { f =>
- val deleteBuilder = mock[DeleteBuilder]
- when(f.curatorClient.delete()).thenReturn(deleteBuilder)
- val g = mock[ChildrenDeletable]
- when(deleteBuilder.guaranteed()).thenReturn(g)
-
- f.stateStore.remove(key)
-
- verify(g).forPath(prefixedKey)
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/sessions/MockSession.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/sessions/MockSession.scala b/server/src/test/scala/com/cloudera/livy/sessions/MockSession.scala
deleted file mode 100644
index 9d129bc..0000000
--- a/server/src/test/scala/com/cloudera/livy/sessions/MockSession.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.sessions
-
-import com.cloudera.livy.LivyConf
-
-class MockSession(id: Int, owner: String, conf: LivyConf) extends Session(id, owner, conf) {
- case class RecoveryMetadata(id: Int) extends Session.RecoveryMetadata()
-
- override val proxyUser = None
-
- override protected def stopSession(): Unit = ()
-
- override def logLines(): IndexedSeq[String] = IndexedSeq()
-
- override def state: SessionState = SessionState.Idle()
-
- override def recoveryMetadata: RecoveryMetadata = RecoveryMetadata(0)
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/sessions/SessionManagerSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/sessions/SessionManagerSpec.scala b/server/src/test/scala/com/cloudera/livy/sessions/SessionManagerSpec.scala
deleted file mode 100644
index ef9b8c1..0000000
--- a/server/src/test/scala/com/cloudera/livy/sessions/SessionManagerSpec.scala
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.sessions
-
-import scala.concurrent.{Await, ExecutionContext, Future}
-import scala.concurrent.duration._
-import scala.language.postfixOps
-import scala.util.{Failure, Try}
-
-import org.mockito.Mockito.{doReturn, never, verify, when}
-import org.scalatest.{FunSpec, Matchers}
-import org.scalatest.concurrent.Eventually._
-import org.scalatest.mock.MockitoSugar.mock
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf}
-import com.cloudera.livy.server.batch.{BatchRecoveryMetadata, BatchSession}
-import com.cloudera.livy.server.interactive.InteractiveSession
-import com.cloudera.livy.server.recovery.SessionStore
-import com.cloudera.livy.sessions.Session.RecoveryMetadata
-
-class SessionManagerSpec extends FunSpec with Matchers with LivyBaseUnitTestSuite {
- implicit def executor: ExecutionContext = ExecutionContext.global
-
- describe("SessionManager") {
- it("should garbage collect old sessions") {
- val livyConf = new LivyConf()
- livyConf.set(LivyConf.SESSION_TIMEOUT, "100ms")
- val manager = new SessionManager[MockSession, RecoveryMetadata](
- livyConf,
- { _ => assert(false).asInstanceOf[MockSession] },
- mock[SessionStore],
- "test",
- Some(Seq.empty))
- val session = manager.register(new MockSession(manager.nextId(), null, livyConf))
- manager.get(session.id).isDefined should be(true)
- eventually(timeout(5 seconds), interval(100 millis)) {
- Await.result(manager.collectGarbage(), Duration.Inf)
- manager.get(session.id) should be(None)
- }
- }
-
- it("batch session should not be gc-ed until application is finished") {
- val sessionId = 24
- val session = mock[BatchSession]
- when(session.id).thenReturn(sessionId)
- when(session.stop()).thenReturn(Future {})
- when(session.lastActivity).thenReturn(System.nanoTime())
-
- val conf = new LivyConf().set(LivyConf.SESSION_STATE_RETAIN_TIME, "1s")
- val sm = new BatchSessionManager(conf, mock[SessionStore], Some(Seq(session)))
- testSessionGC(session, sm)
- }
-
- it("interactive session should not gc-ed if session timeout check is off") {
- val sessionId = 24
- val session = mock[InteractiveSession]
- when(session.id).thenReturn(sessionId)
- when(session.stop()).thenReturn(Future {})
- when(session.lastActivity).thenReturn(System.nanoTime())
-
- val conf = new LivyConf().set(LivyConf.SESSION_TIMEOUT_CHECK, false)
- .set(LivyConf.SESSION_STATE_RETAIN_TIME, "1s")
- val sm = new InteractiveSessionManager(conf, mock[SessionStore], Some(Seq(session)))
- testSessionGC(session, sm)
- }
-
- def testSessionGC(session: Session, sm: SessionManager[_, _]): Unit = {
-
- def changeStateAndCheck(s: SessionState)(fn: SessionManager[_, _] => Unit): Unit = {
- doReturn(s).when(session).state
- Await.result(sm.collectGarbage(), Duration.Inf)
- fn(sm)
- }
-
- // Batch session should not be gc-ed when alive
- for (s <- Seq(SessionState.Running(),
- SessionState.Idle(),
- SessionState.Recovering(),
- SessionState.NotStarted(),
- SessionState.Busy(),
- SessionState.ShuttingDown())) {
- changeStateAndCheck(s) { sm => sm.get(session.id) should be (Some(session)) }
- }
-
- // Stopped session should be gc-ed after retained timeout
- for (s <- Seq(SessionState.Error(),
- SessionState.Success(),
- SessionState.Dead())) {
- eventually(timeout(30 seconds), interval(100 millis)) {
- changeStateAndCheck(s) { sm => sm.get(session.id) should be (None) }
- }
- }
- }
- }
-
- describe("BatchSessionManager") {
- implicit def executor: ExecutionContext = ExecutionContext.global
-
- def makeMetadata(id: Int, appTag: String): BatchRecoveryMetadata = {
- BatchRecoveryMetadata(id, None, appTag, null, None)
- }
-
- def mockSession(id: Int): BatchSession = {
- val session = mock[BatchSession]
- when(session.id).thenReturn(id)
- when(session.stop()).thenReturn(Future {})
- when(session.lastActivity).thenReturn(System.nanoTime())
-
- session
- }
-
- it("should not fail if state store is empty") {
- val conf = new LivyConf()
-
- val sessionStore = mock[SessionStore]
- when(sessionStore.getAllSessions[BatchRecoveryMetadata]("batch"))
- .thenReturn(Seq.empty)
-
- val sm = new BatchSessionManager(conf, sessionStore)
- sm.nextId() shouldBe 0
- }
-
- it("should recover sessions from state store") {
- val conf = new LivyConf()
- conf.set(LivyConf.LIVY_SPARK_MASTER.key, "yarn-cluster")
-
- val sessionType = "batch"
- val nextId = 99
-
- val validMetadata = List(makeMetadata(0, "t1"), makeMetadata(77, "t2")).map(Try(_))
- val invalidMetadata = List(Failure(new Exception("Fake invalid metadata")))
- val sessionStore = mock[SessionStore]
- when(sessionStore.getNextSessionId(sessionType)).thenReturn(nextId)
- when(sessionStore.getAllSessions[BatchRecoveryMetadata](sessionType))
- .thenReturn(validMetadata ++ invalidMetadata)
-
- val sm = new BatchSessionManager(conf, sessionStore)
- sm.nextId() shouldBe nextId
- validMetadata.foreach { m =>
- sm.get(m.get.id) shouldBe defined
- }
- sm.size shouldBe validMetadata.size
- }
-
- it("should delete sessions from state store") {
- val conf = new LivyConf()
-
- val sessionType = "batch"
- val sessionId = 24
- val sessionStore = mock[SessionStore]
- val session = mockSession(sessionId)
-
- val sm = new BatchSessionManager(conf, sessionStore, Some(Seq(session)))
- sm.get(sessionId) shouldBe defined
-
- Await.ready(sm.delete(sessionId).get, 30 seconds)
-
- verify(sessionStore).remove(sessionType, sessionId)
- sm.get(sessionId) shouldBe None
- }
-
- it("should delete sessions on shutdown when recovery is off") {
- val conf = new LivyConf()
- val sessionId = 24
- val sessionStore = mock[SessionStore]
- val session = mockSession(sessionId)
-
- val sm = new BatchSessionManager(conf, sessionStore, Some(Seq(session)))
- sm.get(sessionId) shouldBe defined
- sm.shutdown()
-
- verify(session).stop()
- }
-
- it("should not delete sessions on shutdown with recovery is on") {
- val conf = new LivyConf()
- conf.set(LivyConf.RECOVERY_MODE, SessionManager.SESSION_RECOVERY_MODE_RECOVERY)
-
- val sessionId = 24
- val sessionStore = mock[SessionStore]
- val session = mockSession(sessionId)
-
- val sm = new BatchSessionManager(conf, sessionStore, Some(Seq(session)))
- sm.get(sessionId) shouldBe defined
- sm.shutdown()
-
- verify(session, never).stop()
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/sessions/SessionSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/sessions/SessionSpec.scala b/server/src/test/scala/com/cloudera/livy/sessions/SessionSpec.scala
deleted file mode 100644
index b45a0ed..0000000
--- a/server/src/test/scala/com/cloudera/livy/sessions/SessionSpec.scala
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.sessions
-
-import java.net.URI
-
-import org.scalatest.FunSuite
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf}
-
-class SessionSpec extends FunSuite with LivyBaseUnitTestSuite {
-
- test("use default fs in paths") {
- val conf = new LivyConf(false)
- conf.hadoopConf.set("fs.defaultFS", "dummy:///")
-
- val uris = Seq("http://example.com/foo", "hdfs:/bar", "/baz")
- val expected = Seq(uris(0), uris(1), "dummy:///baz")
- assert(Session.resolveURIs(uris, conf) === expected)
-
- intercept[IllegalArgumentException] {
- Session.resolveURI(new URI("relative_path"), conf)
- }
- }
-
- test("local fs whitelist") {
- val conf = new LivyConf(false)
- conf.set(LivyConf.LOCAL_FS_WHITELIST, "/allowed/,/also_allowed")
-
- Seq("/allowed/file", "/also_allowed/file").foreach { path =>
- assert(Session.resolveURI(new URI(path), conf) === new URI("file://" + path))
- }
-
- Seq("/not_allowed", "/allowed_not_really").foreach { path =>
- intercept[IllegalArgumentException] {
- Session.resolveURI(new URI(path), conf)
- }
- }
- }
-
- test("conf validation and preparation") {
- val conf = new LivyConf(false)
- conf.hadoopConf.set("fs.defaultFS", "dummy:///")
- conf.set(LivyConf.LOCAL_FS_WHITELIST, "/allowed")
-
- // Test baseline.
- assert(Session.prepareConf(Map(), Nil, Nil, Nil, Nil, conf) === Map("spark.master" -> "local"))
-
- // Test validations.
- intercept[IllegalArgumentException] {
- Session.prepareConf(Map("spark.do_not_set" -> "1"), Nil, Nil, Nil, Nil, conf)
- }
- conf.sparkFileLists.foreach { key =>
- intercept[IllegalArgumentException] {
- Session.prepareConf(Map(key -> "file:/not_allowed"), Nil, Nil, Nil, Nil, conf)
- }
- }
- intercept[IllegalArgumentException] {
- Session.prepareConf(Map(), Seq("file:/not_allowed"), Nil, Nil, Nil, conf)
- }
- intercept[IllegalArgumentException] {
- Session.prepareConf(Map(), Nil, Seq("file:/not_allowed"), Nil, Nil, conf)
- }
- intercept[IllegalArgumentException] {
- Session.prepareConf(Map(), Nil, Nil, Seq("file:/not_allowed"), Nil, conf)
- }
- intercept[IllegalArgumentException] {
- Session.prepareConf(Map(), Nil, Nil, Nil, Seq("file:/not_allowed"), conf)
- }
-
- // Test that file lists are merged and resolved.
- val base = "/file1.txt"
- val other = Seq("/file2.txt")
- val expected = Some(Seq("dummy://" + other(0), "dummy://" + base).mkString(","))
-
- val userLists = Seq(LivyConf.SPARK_JARS, LivyConf.SPARK_FILES, LivyConf.SPARK_ARCHIVES,
- LivyConf.SPARK_PY_FILES)
- val baseConf = userLists.map { key => (key -> base) }.toMap
- val result = Session.prepareConf(baseConf, other, other, other, other, conf)
- userLists.foreach { key => assert(result.get(key) === expected) }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/test/scala/com/cloudera/livy/utils/LivySparkUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/com/cloudera/livy/utils/LivySparkUtilsSuite.scala b/server/src/test/scala/com/cloudera/livy/utils/LivySparkUtilsSuite.scala
deleted file mode 100644
index b01194a..0000000
--- a/server/src/test/scala/com/cloudera/livy/utils/LivySparkUtilsSuite.scala
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.utils
-
-import org.scalatest.FunSuite
-import org.scalatest.Matchers
-
-import com.cloudera.livy.{LivyBaseUnitTestSuite, LivyConf}
-import com.cloudera.livy.LivyConf._
-import com.cloudera.livy.server.LivyServer
-
-class LivySparkUtilsSuite extends FunSuite with Matchers with LivyBaseUnitTestSuite {
-
- import LivySparkUtils._
-
- private val livyConf = new LivyConf()
- private val livyConf210 = new LivyConf()
- livyConf210.set(LIVY_SPARK_SCALA_VERSION, "2.10.6")
-
- private val livyConf211 = new LivyConf()
- livyConf211.set(LIVY_SPARK_SCALA_VERSION, "2.11.1")
-
- test("check for SPARK_HOME") {
- testSparkHome(livyConf)
- }
-
- test("check spark-submit version") {
- testSparkSubmit(livyConf)
- }
-
- test("should support Spark 1.6") {
- testSparkVersion("1.6.0")
- testSparkVersion("1.6.1")
- testSparkVersion("1.6.1-SNAPSHOT")
- testSparkVersion("1.6.2")
- testSparkVersion("1.6")
- testSparkVersion("1.6.3.2.5.0-12")
- }
-
- test("should support Spark 2.0.x") {
- testSparkVersion("2.0.0")
- testSparkVersion("2.0.1")
- testSparkVersion("2.0.2")
- testSparkVersion("2.0.3-SNAPSHOT")
- testSparkVersion("2.0.0.2.5.1.0-56") // LIVY-229
- testSparkVersion("2.0")
- testSparkVersion("2.1.0")
- testSparkVersion("2.1.1")
- }
-
- test("should not support Spark older than 1.6") {
- intercept[IllegalArgumentException] { testSparkVersion("1.4.0") }
- intercept[IllegalArgumentException] { testSparkVersion("1.5.0") }
- intercept[IllegalArgumentException] { testSparkVersion("1.5.1") }
- intercept[IllegalArgumentException] { testSparkVersion("1.5.2") }
- intercept[IllegalArgumentException] { testSparkVersion("1.5.0-cdh5.6.1") }
- }
-
- test("should fail on bad version") {
- intercept[IllegalArgumentException] { testSparkVersion("not a version") }
- }
-
- test("should error out if recovery is turned on but master isn't yarn") {
- val livyConf = new LivyConf()
- livyConf.set(LivyConf.LIVY_SPARK_MASTER, "local")
- livyConf.set(LivyConf.RECOVERY_MODE, "recovery")
- val s = new LivyServer()
- intercept[IllegalArgumentException] { s.testRecovery(livyConf) }
- }
-
- test("formatScalaVersion() should format Scala version") {
- formatScalaVersion("2.10.8") shouldBe "2.10"
- formatScalaVersion("2.11.4") shouldBe "2.11"
- formatScalaVersion("2.10") shouldBe "2.10"
- formatScalaVersion("2.10.x.x.x.x") shouldBe "2.10"
-
- // Throw exception for bad Scala version.
- intercept[IllegalArgumentException] { formatScalaVersion("") }
- intercept[IllegalArgumentException] { formatScalaVersion("xxx") }
- }
-
- test("defaultSparkScalaVersion() should return default Scala version") {
- defaultSparkScalaVersion(formatSparkVersion("1.6.0")) shouldBe "2.10"
- defaultSparkScalaVersion(formatSparkVersion("1.6.1")) shouldBe "2.10"
- defaultSparkScalaVersion(formatSparkVersion("1.6.2")) shouldBe "2.10"
- defaultSparkScalaVersion(formatSparkVersion("2.0.0")) shouldBe "2.11"
- defaultSparkScalaVersion(formatSparkVersion("2.0.1")) shouldBe "2.11"
-
- // Throw exception for unsupported Spark version.
- intercept[IllegalArgumentException] { defaultSparkScalaVersion(formatSparkVersion("1.5.0")) }
- }
-
- test("sparkScalaVersion() should use spark-submit detected Scala version.") {
- sparkScalaVersion(formatSparkVersion("2.0.1"), Some("2.10"), livyConf) shouldBe "2.10"
- sparkScalaVersion(formatSparkVersion("1.6.0"), Some("2.11"), livyConf) shouldBe "2.11"
- }
-
- test("sparkScalaVersion() should throw if configured and detected Scala version mismatch.") {
- intercept[IllegalArgumentException] {
- sparkScalaVersion(formatSparkVersion("2.0.1"), Some("2.11"), livyConf210)
- }
- intercept[IllegalArgumentException] {
- sparkScalaVersion(formatSparkVersion("1.6.1"), Some("2.10"), livyConf211)
- }
- }
-
- test("sparkScalaVersion() should use configured Scala version if spark-submit doesn't tell.") {
- sparkScalaVersion(formatSparkVersion("1.6.0"), None, livyConf210) shouldBe "2.10"
- sparkScalaVersion(formatSparkVersion("1.6.2"), None, livyConf210) shouldBe "2.10"
- sparkScalaVersion(formatSparkVersion("2.0.0"), None, livyConf210) shouldBe "2.10"
- sparkScalaVersion(formatSparkVersion("2.0.1"), None, livyConf210) shouldBe "2.10"
- sparkScalaVersion(formatSparkVersion("1.6.0"), None, livyConf211) shouldBe "2.11"
- sparkScalaVersion(formatSparkVersion("1.6.2"), None, livyConf211) shouldBe "2.11"
- sparkScalaVersion(formatSparkVersion("2.0.0"), None, livyConf211) shouldBe "2.11"
- sparkScalaVersion(formatSparkVersion("2.0.1"), None, livyConf211) shouldBe "2.11"
- }
-
- test("sparkScalaVersion() should use default Spark Scala version.") {
- sparkScalaVersion(formatSparkVersion("1.6.0"), None, livyConf) shouldBe "2.10"
- sparkScalaVersion(formatSparkVersion("1.6.2"), None, livyConf) shouldBe "2.10"
- sparkScalaVersion(formatSparkVersion("2.0.0"), None, livyConf) shouldBe "2.11"
- sparkScalaVersion(formatSparkVersion("2.0.1"), None, livyConf) shouldBe "2.11"
- sparkScalaVersion(formatSparkVersion("2.1.0"), None, livyConf) shouldBe "2.11"
- }
-}
[20/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/test/java/com/cloudera/livy/rsc/TestSparkClient.java
----------------------------------------------------------------------
diff --git a/rsc/src/test/java/com/cloudera/livy/rsc/TestSparkClient.java b/rsc/src/test/java/com/cloudera/livy/rsc/TestSparkClient.java
deleted file mode 100644
index 973d012..0000000
--- a/rsc/src/test/java/com/cloudera/livy/rsc/TestSparkClient.java
+++ /dev/null
@@ -1,533 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.jar.JarOutputStream;
-import java.util.zip.ZipEntry;
-
-import org.apache.spark.launcher.SparkLauncher;
-import org.apache.spark.streaming.api.java.JavaStreamingContext;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
-
-import com.cloudera.livy.Job;
-import com.cloudera.livy.JobContext;
-import com.cloudera.livy.JobHandle;
-import com.cloudera.livy.LivyClient;
-import com.cloudera.livy.LivyClientBuilder;
-import com.cloudera.livy.client.common.Serializer;
-import com.cloudera.livy.rsc.rpc.RpcException;
-import com.cloudera.livy.test.jobs.Echo;
-import com.cloudera.livy.test.jobs.Failure;
-import com.cloudera.livy.test.jobs.FileReader;
-import com.cloudera.livy.test.jobs.GetCurrentUser;
-import com.cloudera.livy.test.jobs.SQLGetTweets;
-import com.cloudera.livy.test.jobs.Sleeper;
-import com.cloudera.livy.test.jobs.SmallCount;
-import static com.cloudera.livy.rsc.RSCConf.Entry.*;
-
-public class TestSparkClient {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestSparkClient.class);
-
- // Timeouts are bad... mmmkay.
- private static final long TIMEOUT = 100;
-
- private Properties createConf(boolean local) {
- Properties conf = new Properties();
- if (local) {
- conf.put(CLIENT_IN_PROCESS.key(), "true");
- conf.put(SparkLauncher.SPARK_MASTER, "local");
- conf.put("spark.app.name", "SparkClientSuite Local App");
- } else {
- String classpath = System.getProperty("java.class.path");
- conf.put("spark.app.name", "SparkClientSuite Remote App");
- conf.put(SparkLauncher.DRIVER_MEMORY, "512m");
- conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, classpath);
- conf.put(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH, classpath);
- }
-
- conf.put(LIVY_JARS.key(), "");
- return conf;
- }
-
- @Test
- public void testJobSubmission() throws Exception {
- runTest(true, new TestFunction() {
- @Override
- public void call(LivyClient client) throws Exception {
- JobHandle.Listener<String> listener = newListener();
- JobHandle<String> handle = client.submit(new Echo<>("hello"));
- handle.addListener(listener);
- assertEquals("hello", handle.get(TIMEOUT, TimeUnit.SECONDS));
-
- // Try an invalid state transition on the handle. This ensures that the actual state
- // change we're interested in actually happened, since internally the handle serializes
- // state changes.
- assertFalse(((JobHandleImpl<String>)handle).changeState(JobHandle.State.SENT));
-
- verify(listener).onJobStarted(handle);
- verify(listener).onJobSucceeded(same(handle), eq(handle.get()));
- }
- });
- }
-
- @Test
- public void testSimpleSparkJob() throws Exception {
- runTest(true, new TestFunction() {
- @Override
- public void call(LivyClient client) throws Exception {
- JobHandle<Long> handle = client.submit(new SmallCount(5));
- assertEquals(Long.valueOf(5L), handle.get(TIMEOUT, TimeUnit.SECONDS));
- }
- });
- }
-
- @Test
- public void testJobFailure() throws Exception {
- runTest(true, new TestFunction() {
- @Override
- public void call(LivyClient client) throws Exception {
- JobHandle.Listener<Void> listener = newListener();
- JobHandle<Void> handle = client.submit(new Failure());
- handle.addListener(listener);
- try {
- handle.get(TIMEOUT, TimeUnit.SECONDS);
- fail("Should have thrown an exception.");
- } catch (ExecutionException ee) {
- assertTrue(ee.getCause().getMessage().contains(
- Failure.JobFailureException.class.getName()));
- }
-
- // Try an invalid state transition on the handle. This ensures that the actual state
- // change we're interested in actually happened, since internally the handle serializes
- // state changes.
- assertFalse(((JobHandleImpl<Void>)handle).changeState(JobHandle.State.SENT));
-
- verify(listener).onJobStarted(handle);
- verify(listener).onJobFailed(same(handle), any(Throwable.class));
- }
- });
- }
-
- @Test
- public void testSyncRpc() throws Exception {
- runTest(true, new TestFunction() {
- @Override
- public void call(LivyClient client) throws Exception {
- Future<String> result = client.run(new Echo<>("Hello"));
- assertEquals("Hello", result.get(TIMEOUT, TimeUnit.SECONDS));
- }
- });
- }
-
- @Test
- public void testRemoteClient() throws Exception {
- runTest(false, new TestFunction() {
- @Override
- public void call(LivyClient client) throws Exception {
- JobHandle<Long> handle = client.submit(new SmallCount(5));
- assertEquals(Long.valueOf(5L), handle.get(TIMEOUT, TimeUnit.SECONDS));
- }
- });
- }
-
- @Test
- public void testAddJarsAndFiles() throws Exception {
- runTest(true, new TestFunction() {
- @Override
- public void call(LivyClient client) throws Exception {
- File jar = null;
- File file = null;
-
- try {
- // Test that adding a jar to the remote context makes it show up in the classpath.
- jar = File.createTempFile("test", ".jar");
-
- JarOutputStream jarFile = new JarOutputStream(new FileOutputStream(jar));
- jarFile.putNextEntry(new ZipEntry("test.resource"));
- jarFile.write("test resource".getBytes("UTF-8"));
- jarFile.closeEntry();
- jarFile.close();
-
- client.addJar(new URI("file:" + jar.getAbsolutePath()))
- .get(TIMEOUT, TimeUnit.SECONDS);
-
- // Need to run a Spark job to make sure the jar is added to the class loader. Monitoring
- // SparkContext#addJar() doesn't mean much, we can only be sure jars have been distributed
- // when we run a task after the jar has been added.
- String result = client.submit(new FileReader("test.resource", true))
- .get(TIMEOUT, TimeUnit.SECONDS);
- assertEquals("test resource", result);
-
- // Test that adding a file to the remote context makes it available to executors.
- file = File.createTempFile("test", ".file");
-
- FileOutputStream fileStream = new FileOutputStream(file);
- fileStream.write("test file".getBytes("UTF-8"));
- fileStream.close();
-
- client.addJar(new URI("file:" + file.getAbsolutePath()))
- .get(TIMEOUT, TimeUnit.SECONDS);
-
- // The same applies to files added with "addFile". They're only guaranteed to be available
- // to tasks started after the addFile() call completes.
- result = client.submit(new FileReader(file.getName(), false))
- .get(TIMEOUT, TimeUnit.SECONDS);
- assertEquals("test file", result);
- } finally {
- if (jar != null) {
- jar.delete();
- }
- if (file != null) {
- file.delete();
- }
- }
- }
- });
- }
-
- @Test
- public void testSparkSQLJob() throws Exception {
- runTest(true, new TestFunction() {
- @Override
- void call(LivyClient client) throws Exception {
- JobHandle<List<String>> handle = client.submit(new SQLGetTweets(false));
- List<String> topTweets = handle.get(TIMEOUT, TimeUnit.SECONDS);
- assertEquals(1, topTweets.size());
- assertEquals("[Adventures With Coffee, Code, and Writing.,0]",
- topTweets.get(0));
- }
- });
- }
-
- @Test
- public void testHiveJob() throws Exception {
- runTest(true, new TestFunction() {
- @Override
- void call(LivyClient client) throws Exception {
- JobHandle<List<String>> handle = client.submit(new SQLGetTweets(true));
- List<String> topTweets = handle.get(TIMEOUT, TimeUnit.SECONDS);
- assertEquals(1, topTweets.size());
- assertEquals("[Adventures With Coffee, Code, and Writing.,0]",
- topTweets.get(0));
- }
- });
- }
-
- @Test
- public void testStreamingContext() throws Exception {
- runTest(true, new TestFunction() {
- @Override
- void call(LivyClient client) throws Exception {
- JobHandle<Boolean> handle = client.submit(new SparkStreamingJob());
- Boolean streamingContextCreated = handle.get(TIMEOUT, TimeUnit.SECONDS);
- assertEquals(true, streamingContextCreated);
- }
- });
- }
-
- @Test
- public void testImpersonation() throws Exception {
- final String PROXY = "__proxy__";
-
- runTest(false, new TestFunction() {
- @Override
- void config(Properties conf) {
- conf.put(RSCConf.Entry.PROXY_USER.key(), PROXY);
- }
-
- @Override
- void call(LivyClient client) throws Exception {
- JobHandle<String> handle = client.submit(new GetCurrentUser());
- String userName = handle.get(TIMEOUT, TimeUnit.SECONDS);
- assertEquals(PROXY, userName);
- }
- });
- }
-
- @Test
- public void testConnectToRunningContext() throws Exception {
- runTest(false, new TestFunction() {
- @Override
- void call(LivyClient client) throws Exception {
- URI uri = disconnectClient(client);
-
- // If this tries to create a new context, it will fail because it's missing the
- // needed configuration from createConf().
- LivyClient newClient = new LivyClientBuilder()
- .setURI(uri)
- .build();
-
- try {
- JobHandle<String> handle = newClient.submit(new Echo<>("hello"));
- String result = handle.get(TIMEOUT, TimeUnit.SECONDS);
- assertEquals("hello", result);
- } finally {
- newClient.stop(true);
- }
- }
- });
- }
-
- @Test
- public void testServerIdleTimeout() throws Exception {
- runTest(true, new TestFunction() {
- @Override
- void call(LivyClient client) throws Exception {
- // Close the old client and wait a couple of seconds for the timeout to trigger.
- URI uri = disconnectClient(client);
- TimeUnit.SECONDS.sleep(2);
-
- // Try to connect back with a new client, it should fail. Since there's no API to monitor
- // the connection state, we try to enqueue a long-running job and make sure that it fails,
- // in case the connection actually goes through.
- try {
- LivyClient newClient = new LivyClientBuilder()
- .setURI(uri)
- .build();
-
- try {
- newClient.submit(new Sleeper(TimeUnit.SECONDS.toMillis(TIMEOUT)))
- .get(TIMEOUT, TimeUnit.SECONDS);
- } catch (TimeoutException te) {
- // Shouldn't have gotten here, but catch this so that we stop the client.
- newClient.stop(true);
- }
- fail("Should have failed to contact RSC after idle timeout.");
- } catch (Exception e) {
- // Expected.
- }
- }
-
- @Override
- void config(Properties conf) {
- conf.setProperty(SERVER_IDLE_TIMEOUT.key(), "1s");
- }
- });
- }
-
- @Test
- public void testKillServerWhileSparkSubmitIsRunning() throws Exception {
- Properties conf = createConf(true);
- LivyClient client = null;
- PipedInputStream stubStream = new PipedInputStream(new PipedOutputStream());
- try {
- Process mockSparkSubmit = mock(Process.class);
- when(mockSparkSubmit.getInputStream()).thenReturn(stubStream);
- when(mockSparkSubmit.getErrorStream()).thenReturn(stubStream);
-
- // Block waitFor until process.destroy() is called.
- final CountDownLatch waitForCalled = new CountDownLatch(1);
- when(mockSparkSubmit.waitFor()).thenAnswer(new Answer<Integer>() {
- @Override
- public Integer answer(InvocationOnMock invocation) throws Throwable {
- waitForCalled.await();
- return 0;
- }
- });
-
- // Verify process.destroy() is called.
- final CountDownLatch destroyCalled = new CountDownLatch(1);
- doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- destroyCalled.countDown();
- return null;
- }
- }).when(mockSparkSubmit).destroy();
-
- ContextLauncher.mockSparkSubmit = mockSparkSubmit;
-
- client = new LivyClientBuilder(false).setURI(new URI("rsc:/"))
- .setAll(conf)
- .build();
-
- client.stop(true);
-
- assertTrue(destroyCalled.await(5, TimeUnit.SECONDS));
- waitForCalled.countDown();
- } catch (Exception e) {
- // JUnit prints not so useful backtraces in test summary reports, and we don't see the
- // actual source line of the exception, so print the exception to the logs.
- LOG.error("Test threw exception.", e);
- throw e;
- } finally {
- ContextLauncher.mockSparkSubmit = null;
- stubStream.close();
- if (client != null) {
- client.stop(true);
- }
- }
- }
-
- @Test
- public void testBypass() throws Exception {
- runBypassTest(false);
- }
-
- @Test
- public void testBypassSync() throws Exception {
- runBypassTest(true);
- }
-
- private void runBypassTest(final boolean sync) throws Exception {
- runTest(true, new TestFunction() {
- @Override
- public void call(LivyClient client) throws Exception {
- Serializer s = new Serializer();
- RSCClient lclient = (RSCClient) client;
- ByteBuffer job = s.serialize(new Echo<>("hello"));
- String jobId = lclient.bypass(job, sync);
-
- // Try to fetch the result, trying several times until the timeout runs out, and
- // backing off as attempts fail.
- long deadline = System.nanoTime() + TimeUnit.NANOSECONDS.convert(TIMEOUT, TimeUnit.SECONDS);
- long sleep = 100;
- BypassJobStatus status = null;
- while (System.nanoTime() < deadline) {
- BypassJobStatus currStatus = lclient.getBypassJobStatus(jobId).get(TIMEOUT,
- TimeUnit.SECONDS);
- assertNotEquals(JobHandle.State.CANCELLED, currStatus.state);
- assertNotEquals(JobHandle.State.FAILED, currStatus.state);
- if (currStatus.state.equals(JobHandle.State.SUCCEEDED)) {
- status = currStatus;
- break;
- } else if (deadline - System.nanoTime() > sleep * 2) {
- Thread.sleep(sleep);
- sleep *= 2;
- }
- }
- assertNotNull("Failed to fetch bypass job status.", status);
- assertEquals(JobHandle.State.SUCCEEDED, status.state);
-
- String resultVal = (String) s.deserialize(ByteBuffer.wrap(status.result));
- assertEquals("hello", resultVal);
-
- // After the result is retrieved, the driver should stop tracking the job and release
- // resources associated with it.
- try {
- lclient.getBypassJobStatus(jobId).get(TIMEOUT, TimeUnit.SECONDS);
- fail("Should have failed to retrieve status of released job.");
- } catch (ExecutionException ee) {
- assertTrue(ee.getCause() instanceof RpcException);
- assertTrue(ee.getCause().getMessage().contains(
- "java.util.NoSuchElementException: " + jobId));
- }
- }
- });
- }
-
- private <T> JobHandle.Listener<T> newListener() {
- @SuppressWarnings("unchecked")
- JobHandle.Listener<T> listener =
- (JobHandle.Listener<T>) mock(JobHandle.Listener.class);
- return listener;
- }
-
- private URI disconnectClient(LivyClient client) throws Exception {
- ContextInfo ctx = ((RSCClient) client).getContextInfo();
- URI uri = new URI(String.format("rsc://%s:%s@%s:%d", ctx.clientId, ctx.secret,
- ctx.remoteAddress, ctx.remotePort));
-
- // Close the old client and wait a couple of seconds for the timeout to trigger.
- client.stop(false);
- return uri;
- }
-
- private void runTest(boolean local, TestFunction test) throws Exception {
- Properties conf = createConf(local);
- LivyClient client = null;
- try {
- test.config(conf);
- client = new LivyClientBuilder(false).setURI(new URI("rsc:/"))
- .setAll(conf)
- .build();
-
- // Wait for the context to be up before running the test.
- assertNull(client.submit(new PingJob()).get(TIMEOUT, TimeUnit.SECONDS));
-
- test.call(client);
- } catch (Exception e) {
- // JUnit prints not so useful backtraces in test summary reports, and we don't see the
- // actual source line of the exception, so print the exception to the logs.
- LOG.error("Test threw exception.", e);
- throw e;
- } finally {
- if (client != null) {
- client.stop(true);
- }
- }
- }
-
- /* Since it's hard to test a streaming context, test that a
- * streaming context has been created. Also checks that improper
- * sequence of streaming context calls (i.e create, stop, retrieve)
- * result in a failure.
- */
- private static class SparkStreamingJob implements Job<Boolean> {
- @Override
- public Boolean call(JobContext jc) throws Exception {
- try {
- jc.streamingctx();
- fail("Access before creation: Should throw IllegalStateException");
- } catch (IllegalStateException ex) {
- // Expected.
- }
- try {
- jc.stopStreamingCtx();
- fail("Stop before creation: Should throw IllegalStateException");
- } catch (IllegalStateException ex) {
- // Expected.
- }
- try {
- jc.createStreamingContext(1000L);
- JavaStreamingContext streamingContext = jc.streamingctx();
- jc.stopStreamingCtx();
- jc.streamingctx();
- fail();
- } catch (IllegalStateException ex) {
- // Expected.
- }
-
- jc.createStreamingContext(1000L);
- JavaStreamingContext streamingContext = jc.streamingctx();
- jc.stopStreamingCtx();
- return streamingContext != null;
- }
- }
-
- private abstract static class TestFunction {
- abstract void call(LivyClient client) throws Exception;
- void config(Properties conf) { }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/test/java/com/cloudera/livy/rsc/rpc/TestKryoMessageCodec.java
----------------------------------------------------------------------
diff --git a/rsc/src/test/java/com/cloudera/livy/rsc/rpc/TestKryoMessageCodec.java b/rsc/src/test/java/com/cloudera/livy/rsc/rpc/TestKryoMessageCodec.java
deleted file mode 100644
index a8ede98..0000000
--- a/rsc/src/test/java/com/cloudera/livy/rsc/rpc/TestKryoMessageCodec.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.rpc;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.UnpooledByteBufAllocator;
-import io.netty.channel.embedded.EmbeddedChannel;
-import io.netty.handler.logging.LoggingHandler;
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-public class TestKryoMessageCodec {
-
- private static final String MESSAGE = "Hello World!";
-
- @Test
- public void testKryoCodec() throws Exception {
- List<Object> objects = encodeAndDecode(MESSAGE, null);
- assertEquals(1, objects.size());
- assertEquals(MESSAGE, objects.get(0));
- }
-
- @Test
- public void testFragmentation() throws Exception {
- ByteBuf buf = newBuffer();
- Object[] messages = { "msg1", "msg2" };
- int[] indices = new int[messages.length];
-
- KryoMessageCodec codec = new KryoMessageCodec(0);
-
- for (int i = 0; i < messages.length; i++) {
- codec.encode(null, messages[i], buf);
- indices[i] = buf.writerIndex();
- }
-
- List<Object> objects = new ArrayList<>();
-
- // Don't read enough data for the first message to be decoded.
- codec.decode(null, buf.slice(0, indices[0] - 1), objects);
- assertEquals(0, objects.size());
-
- // Read enough data for just the first message to be decoded.
- codec.decode(null, buf.slice(0, indices[0] + 1), objects);
- assertEquals(1, objects.size());
- }
-
- @Test
- public void testEmbeddedChannel() throws Exception {
- EmbeddedChannel c = new EmbeddedChannel(
- new LoggingHandler(getClass()),
- new KryoMessageCodec(0));
- c.writeAndFlush(MESSAGE);
- assertEquals(1, c.outboundMessages().size());
- assertFalse(MESSAGE.getClass().equals(c.outboundMessages().peek().getClass()));
- c.writeInbound(c.readOutbound());
- assertEquals(1, c.inboundMessages().size());
- assertEquals(MESSAGE, c.readInbound());
- c.close();
- }
-
- @Test
- public void testAutoRegistration() throws Exception {
- KryoMessageCodec codec = new KryoMessageCodec(0, TestMessage.class);
- ByteBuf buf = newBuffer();
- codec.encode(null, new TestMessage(), buf);
-
- List<Object> out = new ArrayList<>();
- codec.decode(null, buf, out);
-
- assertEquals(1, out.size());
- assertTrue(out.get(0) instanceof TestMessage);
- }
-
- @Test
- public void testMaxMessageSize() throws Exception {
- KryoMessageCodec codec = new KryoMessageCodec(1024);
- ByteBuf buf = newBuffer();
- codec.encode(null, new TestMessage(new byte[512]), buf);
-
- try {
- codec.encode(null, new TestMessage(new byte[1025]), buf);
- fail("Should have failed to encode large message.");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().indexOf("maximum allowed size") > 0);
- }
-
- KryoMessageCodec unlimited = new KryoMessageCodec(0);
- buf = newBuffer();
- unlimited.encode(null, new TestMessage(new byte[1025]), buf);
-
- try {
- List<Object> out = new ArrayList<>();
- codec.decode(null, buf, out);
- fail("Should have failed to decode large message.");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().indexOf("maximum allowed size") > 0);
- }
- }
-
- @Test
- public void testNegativeMessageSize() throws Exception {
- KryoMessageCodec codec = new KryoMessageCodec(1024);
- ByteBuf buf = newBuffer();
- buf.writeInt(-1);
-
- try {
- List<Object> out = new ArrayList<>();
- codec.decode(null, buf, out);
- fail("Should have failed to decode message with negative size.");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().indexOf("must be positive") > 0);
- }
- }
-
- @Test
- public void testEncryptionOnly() throws Exception {
- List<Object> objects = Collections.<Object>emptyList();
- try {
- objects = encodeAndDecode(MESSAGE, new TestEncryptionHandler(true, false));
- } catch (Exception e) {
- // Pass.
- }
- // Do this check in case the ciphertext actually makes sense in some way.
- for (Object msg : objects) {
- assertFalse(MESSAGE.equals(objects.get(0)));
- }
- }
-
- @Test
- public void testDecryptionOnly() throws Exception {
- List<Object> objects = Collections.<Object>emptyList();
- try {
- objects = encodeAndDecode(MESSAGE, new TestEncryptionHandler(false, true));
- } catch (Exception e) {
- // Pass.
- }
- // Do this check in case the decrypted plaintext actually makes sense in some way.
- for (Object msg : objects) {
- assertFalse(MESSAGE.equals(objects.get(0)));
- }
- }
-
- @Test
- public void testEncryptDecrypt() throws Exception {
- List<Object> objects = encodeAndDecode(MESSAGE, new TestEncryptionHandler(true, true));
- assertEquals(1, objects.size());
- assertEquals(MESSAGE, objects.get(0));
- }
-
- private List<Object> encodeAndDecode(Object message, KryoMessageCodec.EncryptionHandler eh)
- throws Exception {
- ByteBuf buf = newBuffer();
- KryoMessageCodec codec = new KryoMessageCodec(0);
- codec.setEncryptionHandler(eh);
- codec.encode(null, message, buf);
-
- List<Object> objects = new ArrayList<>();
- codec.decode(null, buf, objects);
- return objects;
- }
-
- private ByteBuf newBuffer() {
- return UnpooledByteBufAllocator.DEFAULT.buffer(1024);
- }
-
- private static class TestMessage {
- byte[] data;
-
- TestMessage() {
- this(null);
- }
-
- TestMessage(byte[] data) {
- this.data = data;
- }
- }
-
- private static class TestEncryptionHandler implements KryoMessageCodec.EncryptionHandler {
-
- private static final byte KEY = 0x42;
-
- private final boolean encrypt;
- private final boolean decrypt;
-
- TestEncryptionHandler(boolean encrypt, boolean decrypt) {
- this.encrypt = encrypt;
- this.decrypt = decrypt;
- }
-
- public byte[] wrap(byte[] data, int offset, int len) throws IOException {
- return encrypt ? transform(data, offset, len) : data;
- }
-
- public byte[] unwrap(byte[] data, int offset, int len) throws IOException {
- return decrypt ? transform(data, offset, len) : data;
- }
-
- public void dispose() throws IOException {
-
- }
-
- private byte[] transform(byte[] data, int offset, int len) {
- byte[] dest = new byte[len];
- for (int i = 0; i < len; i++) {
- dest[i] = (byte) (data[offset + i] ^ KEY);
- }
- return dest;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/test/java/com/cloudera/livy/rsc/rpc/TestRpc.java
----------------------------------------------------------------------
diff --git a/rsc/src/test/java/com/cloudera/livy/rsc/rpc/TestRpc.java b/rsc/src/test/java/com/cloudera/livy/rsc/rpc/TestRpc.java
deleted file mode 100644
index 48abe94..0000000
--- a/rsc/src/test/java/com/cloudera/livy/rsc/rpc/TestRpc.java
+++ /dev/null
@@ -1,337 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.livy.rsc.rpc;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.SocketException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.security.sasl.SaslException;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.embedded.EmbeddedChannel;
-import io.netty.util.concurrent.Future;
-import org.apache.commons.io.IOUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
-
-import com.cloudera.livy.rsc.FutureListener;
-import com.cloudera.livy.rsc.RSCConf;
-import com.cloudera.livy.rsc.Utils;
-import static com.cloudera.livy.rsc.RSCConf.Entry.*;
-
-public class TestRpc {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestRpc.class);
-
- private Collection<Closeable> closeables;
- private RSCConf emptyConfig;
-
- @Before
- public void setUp() {
- closeables = new ArrayList<>();
- emptyConfig = new RSCConf(null);
- }
-
- @After
- public void cleanUp() throws Exception {
- for (Closeable c : closeables) {
- IOUtils.closeQuietly(c);
- }
- }
-
- private <T extends Closeable> T autoClose(T closeable) {
- closeables.add(closeable);
- return closeable;
- }
-
- @Test
- public void testRpcDispatcher() throws Exception {
- Rpc serverRpc = autoClose(Rpc.createEmbedded(new TestDispatcher()));
- Rpc clientRpc = autoClose(Rpc.createEmbedded(new TestDispatcher()));
-
- TestMessage outbound = new TestMessage("Hello World!");
- Future<TestMessage> call = clientRpc.call(outbound, TestMessage.class);
-
- LOG.debug("Transferring messages...");
- transfer(serverRpc, clientRpc);
-
- TestMessage reply = call.get(10, TimeUnit.SECONDS);
- assertEquals(outbound.message, reply.message);
- }
-
- @Test
- public void testClientServer() throws Exception {
- RpcServer server = autoClose(new RpcServer(emptyConfig));
- Rpc[] rpcs = createRpcConnection(server);
- Rpc serverRpc = rpcs[0];
- Rpc client = rpcs[1];
-
- TestMessage outbound = new TestMessage("Hello World!");
- Future<TestMessage> call = client.call(outbound, TestMessage.class);
- TestMessage reply = call.get(10, TimeUnit.SECONDS);
- assertEquals(outbound.message, reply.message);
-
- TestMessage another = new TestMessage("Hello again!");
- Future<TestMessage> anotherCall = client.call(another, TestMessage.class);
- TestMessage anotherReply = anotherCall.get(10, TimeUnit.SECONDS);
- assertEquals(another.message, anotherReply.message);
-
- String errorMsg = "This is an error.";
- try {
- client.call(new ErrorCall(errorMsg)).get(10, TimeUnit.SECONDS);
- } catch (ExecutionException ee) {
- assertTrue(ee.getCause() instanceof RpcException);
- assertTrue(ee.getCause().getMessage().indexOf(errorMsg) >= 0);
- }
-
- // Test from server to client too.
- TestMessage serverMsg = new TestMessage("Hello from the server!");
- Future<TestMessage> serverCall = serverRpc.call(serverMsg, TestMessage.class);
- TestMessage serverReply = serverCall.get(10, TimeUnit.SECONDS);
- assertEquals(serverMsg.message, serverReply.message);
- }
-
- @Test
- public void testBadHello() throws Exception {
- RpcServer server = autoClose(new RpcServer(emptyConfig));
- RpcServer.ClientCallback callback = mock(RpcServer.ClientCallback.class);
-
- server.registerClient("client", "newClient", callback);
- Future<Rpc> clientRpcFuture = Rpc.createClient(emptyConfig, server.getEventLoopGroup(),
- "localhost", server.getPort(), "client", "wrongClient", new TestDispatcher());
-
- try {
- autoClose(clientRpcFuture.get(10, TimeUnit.SECONDS));
- fail("Should have failed to create client with wrong secret.");
- } catch (ExecutionException ee) {
- // On failure, the SASL handler will throw an exception indicating that the SASL
- // negotiation failed.
- assertTrue("Unexpected exception: " + ee.getCause(),
- ee.getCause() instanceof SaslException);
- }
-
- verify(callback, never()).onNewClient(any(Rpc.class));
- }
-
- @Test
- public void testCloseListener() throws Exception {
- RpcServer server = autoClose(new RpcServer(emptyConfig));
- Rpc[] rpcs = createRpcConnection(server);
- Rpc client = rpcs[1];
-
- final AtomicInteger closeCount = new AtomicInteger();
- Utils.addListener(client.getChannel().closeFuture(), new FutureListener<Void>() {
- @Override
- public void onSuccess(Void unused) {
- closeCount.incrementAndGet();
- }
- });
-
- client.close();
- client.close();
- assertEquals(1, closeCount.get());
- }
-
- @Test
- public void testNotDeserializableRpc() throws Exception {
- RpcServer server = autoClose(new RpcServer(emptyConfig));
- Rpc[] rpcs = createRpcConnection(server);
- Rpc client = rpcs[1];
-
- try {
- client.call(new NotDeserializable(42)).get(10, TimeUnit.SECONDS);
- } catch (ExecutionException ee) {
- assertTrue(ee.getCause() instanceof RpcException);
- assertTrue(ee.getCause().getMessage().indexOf("KryoException") >= 0);
- }
- }
-
- @Test
- public void testEncryption() throws Exception {
- RSCConf eConf = new RSCConf(null)
- .setAll(emptyConfig)
- .set(SASL_QOP, Rpc.SASL_AUTH_CONF);
- RpcServer server = autoClose(new RpcServer(eConf));
- Rpc[] rpcs = createRpcConnection(server, eConf);
- Rpc client = rpcs[1];
-
- TestMessage outbound = new TestMessage("Hello World!");
- Future<TestMessage> call = client.call(outbound, TestMessage.class);
- TestMessage reply = call.get(10, TimeUnit.SECONDS);
- assertEquals(outbound.message, reply.message);
- }
-
- @Test
- public void testPortRange() throws Exception {
- String portRange = "a~b";
- emptyConfig.set(LAUNCHER_PORT_RANGE, portRange);
- try {
- autoClose(new RpcServer(emptyConfig));
- } catch (Exception ee) {
- assertTrue(ee instanceof NumberFormatException);
- }
- portRange = "11000";
- emptyConfig.set(LAUNCHER_PORT_RANGE, portRange);
- try {
- autoClose(new RpcServer(emptyConfig));
- } catch (Exception ee) {
- assertTrue(ee instanceof ArrayIndexOutOfBoundsException);
- }
- portRange = "11000~11110";
- emptyConfig.set(LAUNCHER_PORT_RANGE, portRange);
- String [] portRangeData = portRange.split("~");
- int startPort = Integer.parseInt(portRangeData[0]);
- int endPort = Integer.parseInt(portRangeData[1]);
- RpcServer server = autoClose(new RpcServer(emptyConfig));
- assertTrue(startPort <= server.getPort() && server.getPort() <= endPort);
- }
-
- private void transfer(Rpc serverRpc, Rpc clientRpc) {
- EmbeddedChannel client = (EmbeddedChannel) clientRpc.getChannel();
- EmbeddedChannel server = (EmbeddedChannel) serverRpc.getChannel();
-
- server.runPendingTasks();
- client.runPendingTasks();
-
- int count = 0;
- while (!client.outboundMessages().isEmpty()) {
- server.writeInbound(client.readOutbound());
- count++;
- }
- server.flush();
- LOG.debug("Transferred {} outbound client messages.", count);
-
- count = 0;
- while (!server.outboundMessages().isEmpty()) {
- client.writeInbound(server.readOutbound());
- count++;
- }
- client.flush();
- LOG.debug("Transferred {} outbound server messages.", count);
- }
-
- /**
- * Creates a client connection between the server and a client.
- *
- * @return two-tuple (server rpc, client rpc)
- */
- private Rpc[] createRpcConnection(RpcServer server) throws Exception {
- return createRpcConnection(server, emptyConfig);
- }
-
- private Rpc[] createRpcConnection(RpcServer server, RSCConf clientConf)
- throws Exception {
- String secret = server.createSecret();
- ServerRpcCallback callback = new ServerRpcCallback();
- server.registerClient("client", secret, callback);
-
- Future<Rpc> clientRpcFuture = Rpc.createClient(clientConf, server.getEventLoopGroup(),
- "localhost", server.getPort(), "client", secret, new TestDispatcher());
-
- assertTrue("onNewClient() wasn't called.",
- callback.onNewClientCalled.await(10, TimeUnit.SECONDS));
- assertTrue("onSaslComplete() wasn't called.",
- callback.onSaslCompleteCalled.await(10, TimeUnit.SECONDS));
- assertNotNull(callback.client);
- Rpc serverRpc = autoClose(callback.client);
- Rpc clientRpc = autoClose(clientRpcFuture.get(10, TimeUnit.SECONDS));
- return new Rpc[] { serverRpc, clientRpc };
- }
-
- private static class ServerRpcCallback implements RpcServer.ClientCallback {
- final CountDownLatch onNewClientCalled = new CountDownLatch(1);
- final CountDownLatch onSaslCompleteCalled = new CountDownLatch(1);
- Rpc client;
-
- @Override
- public RpcDispatcher onNewClient(Rpc client) {
- this.client = client;
- onNewClientCalled.countDown();
- return new TestDispatcher();
- }
-
- @Override
- public void onSaslComplete(Rpc client) {
- onSaslCompleteCalled.countDown();
- }
-
- }
-
- private static class TestMessage {
-
- final String message;
-
- public TestMessage() {
- this(null);
- }
-
- public TestMessage(String message) {
- this.message = message;
- }
-
- }
-
- private static class ErrorCall {
-
- final String error;
-
- public ErrorCall() {
- this(null);
- }
-
- public ErrorCall(String error) {
- this.error = error;
- }
-
- }
-
- private static class NotDeserializable {
-
- NotDeserializable(int unused) {
-
- }
-
- }
-
- private static class TestDispatcher extends RpcDispatcher {
- protected TestMessage handle(ChannelHandlerContext ctx, TestMessage msg) {
- return msg;
- }
-
- protected void handle(ChannelHandlerContext ctx, ErrorCall msg) {
- throw new IllegalArgumentException(msg.error);
- }
-
- protected void handle(ChannelHandlerContext ctx, NotDeserializable msg) {
- // No op. Shouldn't actually be called, if it is, the test will fail.
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/test/java/org/apache/livy/rsc/TestJobHandle.java
----------------------------------------------------------------------
diff --git a/rsc/src/test/java/org/apache/livy/rsc/TestJobHandle.java b/rsc/src/test/java/org/apache/livy/rsc/TestJobHandle.java
new file mode 100644
index 0000000..e6161ed
--- /dev/null
+++ b/rsc/src/test/java/org/apache/livy/rsc/TestJobHandle.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import io.netty.util.concurrent.Promise;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InOrder;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import org.apache.livy.JobHandle;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestJobHandle {
+
+ @Mock private RSCClient client;
+ @Mock private Promise<Object> promise;
+ @Mock private JobHandle.Listener<Object> listener;
+ @Mock private JobHandle.Listener<Object> listener2;
+
+ @Test
+ public void testStateChanges() throws Exception {
+ JobHandleImpl<Object> handle = new JobHandleImpl<Object>(client, promise, "job");
+ handle.addListener(listener);
+
+ assertTrue(handle.changeState(JobHandle.State.QUEUED));
+ verify(listener).onJobQueued(handle);
+
+ assertTrue(handle.changeState(JobHandle.State.STARTED));
+ verify(listener).onJobStarted(handle);
+
+ assertTrue(handle.changeState(JobHandle.State.CANCELLED));
+ verify(listener).onJobCancelled(handle);
+
+ assertFalse(handle.changeState(JobHandle.State.STARTED));
+ assertFalse(handle.changeState(JobHandle.State.FAILED));
+ assertFalse(handle.changeState(JobHandle.State.SUCCEEDED));
+ }
+
+ @Test
+ public void testFailedJob() throws Exception {
+ JobHandleImpl<Object> handle = new JobHandleImpl<Object>(client, promise, "job");
+ handle.addListener(listener);
+
+ Throwable cause = new Exception();
+ when(promise.cause()).thenReturn(cause);
+
+ assertTrue(handle.changeState(JobHandle.State.FAILED));
+ verify(promise).cause();
+ verify(listener).onJobFailed(handle, cause);
+ }
+
+ @Test
+ public void testSucceededJob() throws Exception {
+ JobHandleImpl<Object> handle = new JobHandleImpl<Object>(client, promise, "job");
+ handle.addListener(listener);
+
+ Object result = new Exception();
+ when(promise.getNow()).thenReturn(result);
+
+ assertTrue(handle.changeState(JobHandle.State.SUCCEEDED));
+ verify(promise).getNow();
+ verify(listener).onJobSucceeded(handle, result);
+ }
+
+ @Test
+ public void testImmediateCallback() throws Exception {
+ JobHandleImpl<Object> handle = new JobHandleImpl<Object>(client, promise, "job");
+ assertTrue(handle.changeState(JobHandle.State.QUEUED));
+ handle.addListener(listener);
+ verify(listener).onJobQueued(handle);
+
+ handle.changeState(JobHandle.State.STARTED);
+ handle.changeState(JobHandle.State.CANCELLED);
+
+ handle.addListener(listener2);
+ verify(listener2).onJobCancelled(same(handle));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java
----------------------------------------------------------------------
diff --git a/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java b/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java
new file mode 100644
index 0000000..0663822
--- /dev/null
+++ b/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java
@@ -0,0 +1,533 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+
+import org.apache.spark.launcher.SparkLauncher;
+import org.apache.spark.streaming.api.java.JavaStreamingContext;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import org.apache.livy.Job;
+import org.apache.livy.JobContext;
+import org.apache.livy.JobHandle;
+import org.apache.livy.LivyClient;
+import org.apache.livy.LivyClientBuilder;
+import org.apache.livy.client.common.Serializer;
+import org.apache.livy.rsc.rpc.RpcException;
+import org.apache.livy.test.jobs.Echo;
+import org.apache.livy.test.jobs.Failure;
+import org.apache.livy.test.jobs.FileReader;
+import org.apache.livy.test.jobs.GetCurrentUser;
+import org.apache.livy.test.jobs.SQLGetTweets;
+import org.apache.livy.test.jobs.Sleeper;
+import org.apache.livy.test.jobs.SmallCount;
+import static org.apache.livy.rsc.RSCConf.Entry.*;
+
+public class TestSparkClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestSparkClient.class);
+
+ // Timeouts are bad... mmmkay.
+ private static final long TIMEOUT = 100;
+
+ private Properties createConf(boolean local) {
+ Properties conf = new Properties();
+ if (local) {
+ conf.put(CLIENT_IN_PROCESS.key(), "true");
+ conf.put(SparkLauncher.SPARK_MASTER, "local");
+ conf.put("spark.app.name", "SparkClientSuite Local App");
+ } else {
+ String classpath = System.getProperty("java.class.path");
+ conf.put("spark.app.name", "SparkClientSuite Remote App");
+ conf.put(SparkLauncher.DRIVER_MEMORY, "512m");
+ conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, classpath);
+ conf.put(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH, classpath);
+ }
+
+ conf.put(LIVY_JARS.key(), "");
+ return conf;
+ }
+
+ @Test
+ public void testJobSubmission() throws Exception {
+ runTest(true, new TestFunction() {
+ @Override
+ public void call(LivyClient client) throws Exception {
+ JobHandle.Listener<String> listener = newListener();
+ JobHandle<String> handle = client.submit(new Echo<>("hello"));
+ handle.addListener(listener);
+ assertEquals("hello", handle.get(TIMEOUT, TimeUnit.SECONDS));
+
+ // Try an invalid state transition on the handle. This ensures that the actual state
+ // change we're interested in actually happened, since internally the handle serializes
+ // state changes.
+ assertFalse(((JobHandleImpl<String>)handle).changeState(JobHandle.State.SENT));
+
+ verify(listener).onJobStarted(handle);
+ verify(listener).onJobSucceeded(same(handle), eq(handle.get()));
+ }
+ });
+ }
+
+ @Test
+ public void testSimpleSparkJob() throws Exception {
+ runTest(true, new TestFunction() {
+ @Override
+ public void call(LivyClient client) throws Exception {
+ JobHandle<Long> handle = client.submit(new SmallCount(5));
+ assertEquals(Long.valueOf(5L), handle.get(TIMEOUT, TimeUnit.SECONDS));
+ }
+ });
+ }
+
+ @Test
+ public void testJobFailure() throws Exception {
+ runTest(true, new TestFunction() {
+ @Override
+ public void call(LivyClient client) throws Exception {
+ JobHandle.Listener<Void> listener = newListener();
+ JobHandle<Void> handle = client.submit(new Failure());
+ handle.addListener(listener);
+ try {
+ handle.get(TIMEOUT, TimeUnit.SECONDS);
+ fail("Should have thrown an exception.");
+ } catch (ExecutionException ee) {
+ assertTrue(ee.getCause().getMessage().contains(
+ Failure.JobFailureException.class.getName()));
+ }
+
+ // Try an invalid state transition on the handle. This ensures that the actual state
+ // change we're interested in actually happened, since internally the handle serializes
+ // state changes.
+ assertFalse(((JobHandleImpl<Void>)handle).changeState(JobHandle.State.SENT));
+
+ verify(listener).onJobStarted(handle);
+ verify(listener).onJobFailed(same(handle), any(Throwable.class));
+ }
+ });
+ }
+
+ @Test
+ public void testSyncRpc() throws Exception {
+ runTest(true, new TestFunction() {
+ @Override
+ public void call(LivyClient client) throws Exception {
+ Future<String> result = client.run(new Echo<>("Hello"));
+ assertEquals("Hello", result.get(TIMEOUT, TimeUnit.SECONDS));
+ }
+ });
+ }
+
+ @Test
+ public void testRemoteClient() throws Exception {
+ runTest(false, new TestFunction() {
+ @Override
+ public void call(LivyClient client) throws Exception {
+ JobHandle<Long> handle = client.submit(new SmallCount(5));
+ assertEquals(Long.valueOf(5L), handle.get(TIMEOUT, TimeUnit.SECONDS));
+ }
+ });
+ }
+
+ @Test
+ public void testAddJarsAndFiles() throws Exception {
+ runTest(true, new TestFunction() {
+ @Override
+ public void call(LivyClient client) throws Exception {
+ File jar = null;
+ File file = null;
+
+ try {
+ // Test that adding a jar to the remote context makes it show up in the classpath.
+ jar = File.createTempFile("test", ".jar");
+
+ JarOutputStream jarFile = new JarOutputStream(new FileOutputStream(jar));
+ jarFile.putNextEntry(new ZipEntry("test.resource"));
+ jarFile.write("test resource".getBytes("UTF-8"));
+ jarFile.closeEntry();
+ jarFile.close();
+
+ client.addJar(new URI("file:" + jar.getAbsolutePath()))
+ .get(TIMEOUT, TimeUnit.SECONDS);
+
+ // Need to run a Spark job to make sure the jar is added to the class loader. Monitoring
+ // SparkContext#addJar() doesn't mean much, we can only be sure jars have been distributed
+ // when we run a task after the jar has been added.
+ String result = client.submit(new FileReader("test.resource", true))
+ .get(TIMEOUT, TimeUnit.SECONDS);
+ assertEquals("test resource", result);
+
+ // Test that adding a file to the remote context makes it available to executors.
+ file = File.createTempFile("test", ".file");
+
+ FileOutputStream fileStream = new FileOutputStream(file);
+ fileStream.write("test file".getBytes("UTF-8"));
+ fileStream.close();
+
+ client.addJar(new URI("file:" + file.getAbsolutePath()))
+ .get(TIMEOUT, TimeUnit.SECONDS);
+
+ // The same applies to files added with "addFile". They're only guaranteed to be available
+ // to tasks started after the addFile() call completes.
+ result = client.submit(new FileReader(file.getName(), false))
+ .get(TIMEOUT, TimeUnit.SECONDS);
+ assertEquals("test file", result);
+ } finally {
+ if (jar != null) {
+ jar.delete();
+ }
+ if (file != null) {
+ file.delete();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testSparkSQLJob() throws Exception {
+ runTest(true, new TestFunction() {
+ @Override
+ void call(LivyClient client) throws Exception {
+ JobHandle<List<String>> handle = client.submit(new SQLGetTweets(false));
+ List<String> topTweets = handle.get(TIMEOUT, TimeUnit.SECONDS);
+ assertEquals(1, topTweets.size());
+ assertEquals("[Adventures With Coffee, Code, and Writing.,0]",
+ topTweets.get(0));
+ }
+ });
+ }
+
+ @Test
+ public void testHiveJob() throws Exception {
+ runTest(true, new TestFunction() {
+ @Override
+ void call(LivyClient client) throws Exception {
+ JobHandle<List<String>> handle = client.submit(new SQLGetTweets(true));
+ List<String> topTweets = handle.get(TIMEOUT, TimeUnit.SECONDS);
+ assertEquals(1, topTweets.size());
+ assertEquals("[Adventures With Coffee, Code, and Writing.,0]",
+ topTweets.get(0));
+ }
+ });
+ }
+
+ @Test
+ public void testStreamingContext() throws Exception {
+ runTest(true, new TestFunction() {
+ @Override
+ void call(LivyClient client) throws Exception {
+ JobHandle<Boolean> handle = client.submit(new SparkStreamingJob());
+ Boolean streamingContextCreated = handle.get(TIMEOUT, TimeUnit.SECONDS);
+ assertEquals(true, streamingContextCreated);
+ }
+ });
+ }
+
+ @Test
+ public void testImpersonation() throws Exception {
+ final String PROXY = "__proxy__";
+
+ runTest(false, new TestFunction() {
+ @Override
+ void config(Properties conf) {
+ conf.put(RSCConf.Entry.PROXY_USER.key(), PROXY);
+ }
+
+ @Override
+ void call(LivyClient client) throws Exception {
+ JobHandle<String> handle = client.submit(new GetCurrentUser());
+ String userName = handle.get(TIMEOUT, TimeUnit.SECONDS);
+ assertEquals(PROXY, userName);
+ }
+ });
+ }
+
+ @Test
+ public void testConnectToRunningContext() throws Exception {
+ runTest(false, new TestFunction() {
+ @Override
+ void call(LivyClient client) throws Exception {
+ URI uri = disconnectClient(client);
+
+ // If this tries to create a new context, it will fail because it's missing the
+ // needed configuration from createConf().
+ LivyClient newClient = new LivyClientBuilder()
+ .setURI(uri)
+ .build();
+
+ try {
+ JobHandle<String> handle = newClient.submit(new Echo<>("hello"));
+ String result = handle.get(TIMEOUT, TimeUnit.SECONDS);
+ assertEquals("hello", result);
+ } finally {
+ newClient.stop(true);
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testServerIdleTimeout() throws Exception {
+ runTest(true, new TestFunction() {
+ @Override
+ void call(LivyClient client) throws Exception {
+ // Close the old client and wait a couple of seconds for the timeout to trigger.
+ URI uri = disconnectClient(client);
+ TimeUnit.SECONDS.sleep(2);
+
+ // Try to connect back with a new client, it should fail. Since there's no API to monitor
+ // the connection state, we try to enqueue a long-running job and make sure that it fails,
+ // in case the connection actually goes through.
+ try {
+ LivyClient newClient = new LivyClientBuilder()
+ .setURI(uri)
+ .build();
+
+ try {
+ newClient.submit(new Sleeper(TimeUnit.SECONDS.toMillis(TIMEOUT)))
+ .get(TIMEOUT, TimeUnit.SECONDS);
+ } catch (TimeoutException te) {
+ // Shouldn't have gotten here, but catch this so that we stop the client.
+ newClient.stop(true);
+ }
+ fail("Should have failed to contact RSC after idle timeout.");
+ } catch (Exception e) {
+ // Expected.
+ }
+ }
+
+ @Override
+ void config(Properties conf) {
+ conf.setProperty(SERVER_IDLE_TIMEOUT.key(), "1s");
+ }
+ });
+ }
+
+ @Test
+ public void testKillServerWhileSparkSubmitIsRunning() throws Exception {
+ Properties conf = createConf(true);
+ LivyClient client = null;
+ PipedInputStream stubStream = new PipedInputStream(new PipedOutputStream());
+ try {
+ Process mockSparkSubmit = mock(Process.class);
+ when(mockSparkSubmit.getInputStream()).thenReturn(stubStream);
+ when(mockSparkSubmit.getErrorStream()).thenReturn(stubStream);
+
+ // Block waitFor until process.destroy() is called.
+ final CountDownLatch waitForCalled = new CountDownLatch(1);
+ when(mockSparkSubmit.waitFor()).thenAnswer(new Answer<Integer>() {
+ @Override
+ public Integer answer(InvocationOnMock invocation) throws Throwable {
+ waitForCalled.await();
+ return 0;
+ }
+ });
+
+ // Verify process.destroy() is called.
+ final CountDownLatch destroyCalled = new CountDownLatch(1);
+ doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ destroyCalled.countDown();
+ return null;
+ }
+ }).when(mockSparkSubmit).destroy();
+
+ ContextLauncher.mockSparkSubmit = mockSparkSubmit;
+
+ client = new LivyClientBuilder(false).setURI(new URI("rsc:/"))
+ .setAll(conf)
+ .build();
+
+ client.stop(true);
+
+ assertTrue(destroyCalled.await(5, TimeUnit.SECONDS));
+ waitForCalled.countDown();
+ } catch (Exception e) {
+ // JUnit prints not so useful backtraces in test summary reports, and we don't see the
+ // actual source line of the exception, so print the exception to the logs.
+ LOG.error("Test threw exception.", e);
+ throw e;
+ } finally {
+ ContextLauncher.mockSparkSubmit = null;
+ stubStream.close();
+ if (client != null) {
+ client.stop(true);
+ }
+ }
+ }
+
+ @Test
+ public void testBypass() throws Exception {
+ runBypassTest(false);
+ }
+
+ @Test
+ public void testBypassSync() throws Exception {
+ runBypassTest(true);
+ }
+
+ private void runBypassTest(final boolean sync) throws Exception {
+ runTest(true, new TestFunction() {
+ @Override
+ public void call(LivyClient client) throws Exception {
+ Serializer s = new Serializer();
+ RSCClient lclient = (RSCClient) client;
+ ByteBuffer job = s.serialize(new Echo<>("hello"));
+ String jobId = lclient.bypass(job, sync);
+
+ // Try to fetch the result, trying several times until the timeout runs out, and
+ // backing off as attempts fail.
+ long deadline = System.nanoTime() + TimeUnit.NANOSECONDS.convert(TIMEOUT, TimeUnit.SECONDS);
+ long sleep = 100;
+ BypassJobStatus status = null;
+ while (System.nanoTime() < deadline) {
+ BypassJobStatus currStatus = lclient.getBypassJobStatus(jobId).get(TIMEOUT,
+ TimeUnit.SECONDS);
+ assertNotEquals(JobHandle.State.CANCELLED, currStatus.state);
+ assertNotEquals(JobHandle.State.FAILED, currStatus.state);
+ if (currStatus.state.equals(JobHandle.State.SUCCEEDED)) {
+ status = currStatus;
+ break;
+ } else if (deadline - System.nanoTime() > sleep * 2) {
+ Thread.sleep(sleep);
+ sleep *= 2;
+ }
+ }
+ assertNotNull("Failed to fetch bypass job status.", status);
+ assertEquals(JobHandle.State.SUCCEEDED, status.state);
+
+ String resultVal = (String) s.deserialize(ByteBuffer.wrap(status.result));
+ assertEquals("hello", resultVal);
+
+ // After the result is retrieved, the driver should stop tracking the job and release
+ // resources associated with it.
+ try {
+ lclient.getBypassJobStatus(jobId).get(TIMEOUT, TimeUnit.SECONDS);
+ fail("Should have failed to retrieve status of released job.");
+ } catch (ExecutionException ee) {
+ assertTrue(ee.getCause() instanceof RpcException);
+ assertTrue(ee.getCause().getMessage().contains(
+ "java.util.NoSuchElementException: " + jobId));
+ }
+ }
+ });
+ }
+
+ private <T> JobHandle.Listener<T> newListener() {
+ @SuppressWarnings("unchecked")
+ JobHandle.Listener<T> listener =
+ (JobHandle.Listener<T>) mock(JobHandle.Listener.class);
+ return listener;
+ }
+
+ private URI disconnectClient(LivyClient client) throws Exception {
+ ContextInfo ctx = ((RSCClient) client).getContextInfo();
+ URI uri = new URI(String.format("rsc://%s:%s@%s:%d", ctx.clientId, ctx.secret,
+ ctx.remoteAddress, ctx.remotePort));
+
+ // Close the old client and wait a couple of seconds for the timeout to trigger.
+ client.stop(false);
+ return uri;
+ }
+
+ private void runTest(boolean local, TestFunction test) throws Exception {
+ Properties conf = createConf(local);
+ LivyClient client = null;
+ try {
+ test.config(conf);
+ client = new LivyClientBuilder(false).setURI(new URI("rsc:/"))
+ .setAll(conf)
+ .build();
+
+ // Wait for the context to be up before running the test.
+ assertNull(client.submit(new PingJob()).get(TIMEOUT, TimeUnit.SECONDS));
+
+ test.call(client);
+ } catch (Exception e) {
+ // JUnit prints not so useful backtraces in test summary reports, and we don't see the
+ // actual source line of the exception, so print the exception to the logs.
+ LOG.error("Test threw exception.", e);
+ throw e;
+ } finally {
+ if (client != null) {
+ client.stop(true);
+ }
+ }
+ }
+
+ /* Since it's hard to test a streaming context, test that a
+ * streaming context has been created. Also checks that improper
+ * sequence of streaming context calls (i.e create, stop, retrieve)
+ * result in a failure.
+ */
+ private static class SparkStreamingJob implements Job<Boolean> {
+ @Override
+ public Boolean call(JobContext jc) throws Exception {
+ try {
+ jc.streamingctx();
+ fail("Access before creation: Should throw IllegalStateException");
+ } catch (IllegalStateException ex) {
+ // Expected.
+ }
+ try {
+ jc.stopStreamingCtx();
+ fail("Stop before creation: Should throw IllegalStateException");
+ } catch (IllegalStateException ex) {
+ // Expected.
+ }
+ try {
+ jc.createStreamingContext(1000L);
+ JavaStreamingContext streamingContext = jc.streamingctx();
+ jc.stopStreamingCtx();
+ jc.streamingctx();
+ fail();
+ } catch (IllegalStateException ex) {
+ // Expected.
+ }
+
+ jc.createStreamingContext(1000L);
+ JavaStreamingContext streamingContext = jc.streamingctx();
+ jc.stopStreamingCtx();
+ return streamingContext != null;
+ }
+ }
+
+ private abstract static class TestFunction {
+ abstract void call(LivyClient client) throws Exception;
+ void config(Properties conf) { }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/test/java/org/apache/livy/rsc/rpc/TestKryoMessageCodec.java
----------------------------------------------------------------------
diff --git a/rsc/src/test/java/org/apache/livy/rsc/rpc/TestKryoMessageCodec.java b/rsc/src/test/java/org/apache/livy/rsc/rpc/TestKryoMessageCodec.java
new file mode 100644
index 0000000..a09ac43
--- /dev/null
+++ b/rsc/src/test/java/org/apache/livy/rsc/rpc/TestKryoMessageCodec.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.rpc;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.UnpooledByteBufAllocator;
+import io.netty.channel.embedded.EmbeddedChannel;
+import io.netty.handler.logging.LoggingHandler;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestKryoMessageCodec {
+
+ private static final String MESSAGE = "Hello World!";
+
+ @Test
+ public void testKryoCodec() throws Exception {
+ List<Object> objects = encodeAndDecode(MESSAGE, null);
+ assertEquals(1, objects.size());
+ assertEquals(MESSAGE, objects.get(0));
+ }
+
+ @Test
+ public void testFragmentation() throws Exception {
+ ByteBuf buf = newBuffer();
+ Object[] messages = { "msg1", "msg2" };
+ int[] indices = new int[messages.length];
+
+ KryoMessageCodec codec = new KryoMessageCodec(0);
+
+ for (int i = 0; i < messages.length; i++) {
+ codec.encode(null, messages[i], buf);
+ indices[i] = buf.writerIndex();
+ }
+
+ List<Object> objects = new ArrayList<>();
+
+ // Don't read enough data for the first message to be decoded.
+ codec.decode(null, buf.slice(0, indices[0] - 1), objects);
+ assertEquals(0, objects.size());
+
+ // Read enough data for just the first message to be decoded.
+ codec.decode(null, buf.slice(0, indices[0] + 1), objects);
+ assertEquals(1, objects.size());
+ }
+
+ @Test
+ public void testEmbeddedChannel() throws Exception {
+ EmbeddedChannel c = new EmbeddedChannel(
+ new LoggingHandler(getClass()),
+ new KryoMessageCodec(0));
+ c.writeAndFlush(MESSAGE);
+ assertEquals(1, c.outboundMessages().size());
+ assertFalse(MESSAGE.getClass().equals(c.outboundMessages().peek().getClass()));
+ c.writeInbound(c.readOutbound());
+ assertEquals(1, c.inboundMessages().size());
+ assertEquals(MESSAGE, c.readInbound());
+ c.close();
+ }
+
+ @Test
+ public void testAutoRegistration() throws Exception {
+ KryoMessageCodec codec = new KryoMessageCodec(0, TestMessage.class);
+ ByteBuf buf = newBuffer();
+ codec.encode(null, new TestMessage(), buf);
+
+ List<Object> out = new ArrayList<>();
+ codec.decode(null, buf, out);
+
+ assertEquals(1, out.size());
+ assertTrue(out.get(0) instanceof TestMessage);
+ }
+
+ @Test
+ public void testMaxMessageSize() throws Exception {
+ KryoMessageCodec codec = new KryoMessageCodec(1024);
+ ByteBuf buf = newBuffer();
+ codec.encode(null, new TestMessage(new byte[512]), buf);
+
+ try {
+ codec.encode(null, new TestMessage(new byte[1025]), buf);
+ fail("Should have failed to encode large message.");
+ } catch (IllegalArgumentException e) {
+ assertTrue(e.getMessage().indexOf("maximum allowed size") > 0);
+ }
+
+ KryoMessageCodec unlimited = new KryoMessageCodec(0);
+ buf = newBuffer();
+ unlimited.encode(null, new TestMessage(new byte[1025]), buf);
+
+ try {
+ List<Object> out = new ArrayList<>();
+ codec.decode(null, buf, out);
+ fail("Should have failed to decode large message.");
+ } catch (IllegalArgumentException e) {
+ assertTrue(e.getMessage().indexOf("maximum allowed size") > 0);
+ }
+ }
+
+ @Test
+ public void testNegativeMessageSize() throws Exception {
+ KryoMessageCodec codec = new KryoMessageCodec(1024);
+ ByteBuf buf = newBuffer();
+ buf.writeInt(-1);
+
+ try {
+ List<Object> out = new ArrayList<>();
+ codec.decode(null, buf, out);
+ fail("Should have failed to decode message with negative size.");
+ } catch (IllegalArgumentException e) {
+ assertTrue(e.getMessage().indexOf("must be positive") > 0);
+ }
+ }
+
+ @Test
+ public void testEncryptionOnly() throws Exception {
+ List<Object> objects = Collections.<Object>emptyList();
+ try {
+ objects = encodeAndDecode(MESSAGE, new TestEncryptionHandler(true, false));
+ } catch (Exception e) {
+ // Pass.
+ }
+ // Do this check in case the ciphertext actually makes sense in some way.
+ for (Object msg : objects) {
+ assertFalse(MESSAGE.equals(objects.get(0)));
+ }
+ }
+
+ @Test
+ public void testDecryptionOnly() throws Exception {
+ List<Object> objects = Collections.<Object>emptyList();
+ try {
+ objects = encodeAndDecode(MESSAGE, new TestEncryptionHandler(false, true));
+ } catch (Exception e) {
+ // Pass.
+ }
+ // Do this check in case the decrypted plaintext actually makes sense in some way.
+ for (Object msg : objects) {
+ assertFalse(MESSAGE.equals(objects.get(0)));
+ }
+ }
+
+ @Test
+ public void testEncryptDecrypt() throws Exception {
+ List<Object> objects = encodeAndDecode(MESSAGE, new TestEncryptionHandler(true, true));
+ assertEquals(1, objects.size());
+ assertEquals(MESSAGE, objects.get(0));
+ }
+
+ private List<Object> encodeAndDecode(Object message, KryoMessageCodec.EncryptionHandler eh)
+ throws Exception {
+ ByteBuf buf = newBuffer();
+ KryoMessageCodec codec = new KryoMessageCodec(0);
+ codec.setEncryptionHandler(eh);
+ codec.encode(null, message, buf);
+
+ List<Object> objects = new ArrayList<>();
+ codec.decode(null, buf, objects);
+ return objects;
+ }
+
+ private ByteBuf newBuffer() {
+ return UnpooledByteBufAllocator.DEFAULT.buffer(1024);
+ }
+
+ private static class TestMessage {
+ byte[] data;
+
+ TestMessage() {
+ this(null);
+ }
+
+ TestMessage(byte[] data) {
+ this.data = data;
+ }
+ }
+
+ private static class TestEncryptionHandler implements KryoMessageCodec.EncryptionHandler {
+
+ private static final byte KEY = 0x42;
+
+ private final boolean encrypt;
+ private final boolean decrypt;
+
+ TestEncryptionHandler(boolean encrypt, boolean decrypt) {
+ this.encrypt = encrypt;
+ this.decrypt = decrypt;
+ }
+
+ public byte[] wrap(byte[] data, int offset, int len) throws IOException {
+ return encrypt ? transform(data, offset, len) : data;
+ }
+
+ public byte[] unwrap(byte[] data, int offset, int len) throws IOException {
+ return decrypt ? transform(data, offset, len) : data;
+ }
+
+ public void dispose() throws IOException {
+
+ }
+
+ private byte[] transform(byte[] data, int offset, int len) {
+ byte[] dest = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dest[i] = (byte) (data[offset + i] ^ KEY);
+ }
+ return dest;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/rsc/src/test/java/org/apache/livy/rsc/rpc/TestRpc.java
----------------------------------------------------------------------
diff --git a/rsc/src/test/java/org/apache/livy/rsc/rpc/TestRpc.java b/rsc/src/test/java/org/apache/livy/rsc/rpc/TestRpc.java
new file mode 100644
index 0000000..8967906
--- /dev/null
+++ b/rsc/src/test/java/org/apache/livy/rsc/rpc/TestRpc.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.rsc.rpc;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.SocketException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.security.sasl.SaslException;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.embedded.EmbeddedChannel;
+import io.netty.util.concurrent.Future;
+import org.apache.commons.io.IOUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import org.apache.livy.rsc.FutureListener;
+import org.apache.livy.rsc.RSCConf;
+import org.apache.livy.rsc.Utils;
+import static org.apache.livy.rsc.RSCConf.Entry.*;
+
+public class TestRpc {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestRpc.class);
+
+ private Collection<Closeable> closeables;
+ private RSCConf emptyConfig;
+
+ @Before
+ public void setUp() {
+ closeables = new ArrayList<>();
+ emptyConfig = new RSCConf(null);
+ }
+
+ @After
+ public void cleanUp() throws Exception {
+ for (Closeable c : closeables) {
+ IOUtils.closeQuietly(c);
+ }
+ }
+
+ private <T extends Closeable> T autoClose(T closeable) {
+ closeables.add(closeable);
+ return closeable;
+ }
+
+ @Test
+ public void testRpcDispatcher() throws Exception {
+ Rpc serverRpc = autoClose(Rpc.createEmbedded(new TestDispatcher()));
+ Rpc clientRpc = autoClose(Rpc.createEmbedded(new TestDispatcher()));
+
+ TestMessage outbound = new TestMessage("Hello World!");
+ Future<TestMessage> call = clientRpc.call(outbound, TestMessage.class);
+
+ LOG.debug("Transferring messages...");
+ transfer(serverRpc, clientRpc);
+
+ TestMessage reply = call.get(10, TimeUnit.SECONDS);
+ assertEquals(outbound.message, reply.message);
+ }
+
+ @Test
+ public void testClientServer() throws Exception {
+ RpcServer server = autoClose(new RpcServer(emptyConfig));
+ Rpc[] rpcs = createRpcConnection(server);
+ Rpc serverRpc = rpcs[0];
+ Rpc client = rpcs[1];
+
+ TestMessage outbound = new TestMessage("Hello World!");
+ Future<TestMessage> call = client.call(outbound, TestMessage.class);
+ TestMessage reply = call.get(10, TimeUnit.SECONDS);
+ assertEquals(outbound.message, reply.message);
+
+ TestMessage another = new TestMessage("Hello again!");
+ Future<TestMessage> anotherCall = client.call(another, TestMessage.class);
+ TestMessage anotherReply = anotherCall.get(10, TimeUnit.SECONDS);
+ assertEquals(another.message, anotherReply.message);
+
+ String errorMsg = "This is an error.";
+ try {
+ client.call(new ErrorCall(errorMsg)).get(10, TimeUnit.SECONDS);
+ } catch (ExecutionException ee) {
+ assertTrue(ee.getCause() instanceof RpcException);
+ assertTrue(ee.getCause().getMessage().indexOf(errorMsg) >= 0);
+ }
+
+ // Test from server to client too.
+ TestMessage serverMsg = new TestMessage("Hello from the server!");
+ Future<TestMessage> serverCall = serverRpc.call(serverMsg, TestMessage.class);
+ TestMessage serverReply = serverCall.get(10, TimeUnit.SECONDS);
+ assertEquals(serverMsg.message, serverReply.message);
+ }
+
+ @Test
+ public void testBadHello() throws Exception {
+ RpcServer server = autoClose(new RpcServer(emptyConfig));
+ RpcServer.ClientCallback callback = mock(RpcServer.ClientCallback.class);
+
+ server.registerClient("client", "newClient", callback);
+ Future<Rpc> clientRpcFuture = Rpc.createClient(emptyConfig, server.getEventLoopGroup(),
+ "localhost", server.getPort(), "client", "wrongClient", new TestDispatcher());
+
+ try {
+ autoClose(clientRpcFuture.get(10, TimeUnit.SECONDS));
+ fail("Should have failed to create client with wrong secret.");
+ } catch (ExecutionException ee) {
+ // On failure, the SASL handler will throw an exception indicating that the SASL
+ // negotiation failed.
+ assertTrue("Unexpected exception: " + ee.getCause(),
+ ee.getCause() instanceof SaslException);
+ }
+
+ verify(callback, never()).onNewClient(any(Rpc.class));
+ }
+
+ @Test
+ public void testCloseListener() throws Exception {
+ RpcServer server = autoClose(new RpcServer(emptyConfig));
+ Rpc[] rpcs = createRpcConnection(server);
+ Rpc client = rpcs[1];
+
+ final AtomicInteger closeCount = new AtomicInteger();
+ Utils.addListener(client.getChannel().closeFuture(), new FutureListener<Void>() {
+ @Override
+ public void onSuccess(Void unused) {
+ closeCount.incrementAndGet();
+ }
+ });
+
+ client.close();
+ client.close();
+ assertEquals(1, closeCount.get());
+ }
+
+ @Test
+ public void testNotDeserializableRpc() throws Exception {
+ RpcServer server = autoClose(new RpcServer(emptyConfig));
+ Rpc[] rpcs = createRpcConnection(server);
+ Rpc client = rpcs[1];
+
+ try {
+ client.call(new NotDeserializable(42)).get(10, TimeUnit.SECONDS);
+ } catch (ExecutionException ee) {
+ assertTrue(ee.getCause() instanceof RpcException);
+ assertTrue(ee.getCause().getMessage().indexOf("KryoException") >= 0);
+ }
+ }
+
+ @Test
+ public void testEncryption() throws Exception {
+ RSCConf eConf = new RSCConf(null)
+ .setAll(emptyConfig)
+ .set(SASL_QOP, Rpc.SASL_AUTH_CONF);
+ RpcServer server = autoClose(new RpcServer(eConf));
+ Rpc[] rpcs = createRpcConnection(server, eConf);
+ Rpc client = rpcs[1];
+
+ TestMessage outbound = new TestMessage("Hello World!");
+ Future<TestMessage> call = client.call(outbound, TestMessage.class);
+ TestMessage reply = call.get(10, TimeUnit.SECONDS);
+ assertEquals(outbound.message, reply.message);
+ }
+
+ @Test
+ public void testPortRange() throws Exception {
+ String portRange = "a~b";
+ emptyConfig.set(LAUNCHER_PORT_RANGE, portRange);
+ try {
+ autoClose(new RpcServer(emptyConfig));
+ } catch (Exception ee) {
+ assertTrue(ee instanceof NumberFormatException);
+ }
+ portRange = "11000";
+ emptyConfig.set(LAUNCHER_PORT_RANGE, portRange);
+ try {
+ autoClose(new RpcServer(emptyConfig));
+ } catch (Exception ee) {
+ assertTrue(ee instanceof ArrayIndexOutOfBoundsException);
+ }
+ portRange = "11000~11110";
+ emptyConfig.set(LAUNCHER_PORT_RANGE, portRange);
+ String [] portRangeData = portRange.split("~");
+ int startPort = Integer.parseInt(portRangeData[0]);
+ int endPort = Integer.parseInt(portRangeData[1]);
+ RpcServer server = autoClose(new RpcServer(emptyConfig));
+ assertTrue(startPort <= server.getPort() && server.getPort() <= endPort);
+ }
+
+ private void transfer(Rpc serverRpc, Rpc clientRpc) {
+ EmbeddedChannel client = (EmbeddedChannel) clientRpc.getChannel();
+ EmbeddedChannel server = (EmbeddedChannel) serverRpc.getChannel();
+
+ server.runPendingTasks();
+ client.runPendingTasks();
+
+ int count = 0;
+ while (!client.outboundMessages().isEmpty()) {
+ server.writeInbound(client.readOutbound());
+ count++;
+ }
+ server.flush();
+ LOG.debug("Transferred {} outbound client messages.", count);
+
+ count = 0;
+ while (!server.outboundMessages().isEmpty()) {
+ client.writeInbound(server.readOutbound());
+ count++;
+ }
+ client.flush();
+ LOG.debug("Transferred {} outbound server messages.", count);
+ }
+
+ /**
+ * Creates a client connection between the server and a client.
+ *
+ * @return two-tuple (server rpc, client rpc)
+ */
+ private Rpc[] createRpcConnection(RpcServer server) throws Exception {
+ return createRpcConnection(server, emptyConfig);
+ }
+
+ private Rpc[] createRpcConnection(RpcServer server, RSCConf clientConf)
+ throws Exception {
+ String secret = server.createSecret();
+ ServerRpcCallback callback = new ServerRpcCallback();
+ server.registerClient("client", secret, callback);
+
+ Future<Rpc> clientRpcFuture = Rpc.createClient(clientConf, server.getEventLoopGroup(),
+ "localhost", server.getPort(), "client", secret, new TestDispatcher());
+
+ assertTrue("onNewClient() wasn't called.",
+ callback.onNewClientCalled.await(10, TimeUnit.SECONDS));
+ assertTrue("onSaslComplete() wasn't called.",
+ callback.onSaslCompleteCalled.await(10, TimeUnit.SECONDS));
+ assertNotNull(callback.client);
+ Rpc serverRpc = autoClose(callback.client);
+ Rpc clientRpc = autoClose(clientRpcFuture.get(10, TimeUnit.SECONDS));
+ return new Rpc[] { serverRpc, clientRpc };
+ }
+
+ private static class ServerRpcCallback implements RpcServer.ClientCallback {
+ final CountDownLatch onNewClientCalled = new CountDownLatch(1);
+ final CountDownLatch onSaslCompleteCalled = new CountDownLatch(1);
+ Rpc client;
+
+ @Override
+ public RpcDispatcher onNewClient(Rpc client) {
+ this.client = client;
+ onNewClientCalled.countDown();
+ return new TestDispatcher();
+ }
+
+ @Override
+ public void onSaslComplete(Rpc client) {
+ onSaslCompleteCalled.countDown();
+ }
+
+ }
+
+ private static class TestMessage {
+
+ final String message;
+
+ public TestMessage() {
+ this(null);
+ }
+
+ public TestMessage(String message) {
+ this.message = message;
+ }
+
+ }
+
+ private static class ErrorCall {
+
+ final String error;
+
+ public ErrorCall() {
+ this(null);
+ }
+
+ public ErrorCall(String error) {
+ this.error = error;
+ }
+
+ }
+
+ private static class NotDeserializable {
+
+ NotDeserializable(int unused) {
+
+ }
+
+ }
+
+ private static class TestDispatcher extends RpcDispatcher {
+ protected TestMessage handle(ChannelHandlerContext ctx, TestMessage msg) {
+ return msg;
+ }
+
+ protected void handle(ChannelHandlerContext ctx, ErrorCall msg) {
+ throw new IllegalArgumentException(msg.error);
+ }
+
+ protected void handle(ChannelHandlerContext ctx, NotDeserializable msg) {
+ // No op. Shouldn't actually be called, if it is, the test will fail.
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/pom.xml
----------------------------------------------------------------------
diff --git a/scala-api/pom.xml b/scala-api/pom.xml
index e3e63b0..7e4a4b2 100644
--- a/scala-api/pom.xml
+++ b/scala-api/pom.xml
@@ -21,37 +21,37 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>multi-scala-project-root</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../scala/pom.xml</relativePath>
</parent>
<artifactId>livy-scala-api-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<dependencies>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-api</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-rsc</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-core_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-core_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/scala-2.10/pom.xml
----------------------------------------------------------------------
diff --git a/scala-api/scala-2.10/pom.xml b/scala-api/scala-2.10/pom.xml
index e509fe4..96f6bf8 100644
--- a/scala-api/scala-2.10/pom.xml
+++ b/scala-api/scala-2.10/pom.xml
@@ -17,15 +17,15 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-scala-api_2.10</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-scala-api-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/scala-api/scala-2.11/pom.xml
----------------------------------------------------------------------
diff --git a/scala-api/scala-2.11/pom.xml b/scala-api/scala-2.11/pom.xml
index 1b0509a..3690347 100644
--- a/scala-api/scala-2.11/pom.xml
+++ b/scala-api/scala-2.11/pom.xml
@@ -17,15 +17,15 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-scala-api_2.11</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
- <groupId>com.cloudera.livy</groupId>
+ <groupId>org.apache.livy</groupId>
<artifactId>livy-scala-api-parent</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
[13/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/bootstrap.min.css
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/bootstrap.min.css b/server/src/main/resources/org/apache/livy/server/ui/static/bootstrap.min.css
new file mode 100755
index 0000000..72cc3ab
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/bootstrap.min.css
@@ -0,0 +1,14 @@
+/*!
+ * Bootstrap v3.3.7 (http://getbootstrap.com)
+ * Copyright 2011-2017 Twitter, Inc.
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+ */
+
+/*!
+ * Generated using the Bootstrap Customizer (http://getbootstrap.com/customize/?id=794c6d582814e1084501e746ef56e328)
+ * Config saved to config.json and https://gist.github.com/794c6d582814e1084501e746ef56e328
+ *//*!
+ * Bootstrap v3.3.7 (http://getbootstrap.com)
+ * Copyright 2011-2016 Twitter, Inc.
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+ *//*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,menu,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background-color:transparent}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:bold}dfn{font-style:italic}h1{font-size:2em;margin:0.67em 0}mark{background:#ff0;color:#000}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{border:0}svg:not(:root){overflow:hidden}figure{margin:1em 40px}hr{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;height:0}pre{overflow:auto}code,kbd,pre,samp{font-family:monospace, monospa
ce;font-size:1em}button,input,optgroup,select,textarea{color:inherit;font:inherit;margin:0}button{overflow:visible}button,select{text-transform:none}button,html input[type="button"],input[type="reset"],input[type="submit"]{-webkit-appearance:button;cursor:pointer}button[disabled],html input[disabled]{cursor:default}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}input{line-height:normal}input[type="checkbox"],input[type="radio"]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;padding:0}input[type="number"]::-webkit-inner-spin-button,input[type="number"]::-webkit-outer-spin-button{height:auto}input[type="search"]{-webkit-appearance:textfield;-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box}input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-decoration{-webkit-appearance:none}fieldset{border:1px solid #c0c0c0;margin:0 2px;padding:0.35em 0.625em 0.75em}legend{border:0
;padding:0}textarea{overflow:auto}optgroup{font-weight:bold}table{border-collapse:collapse;border-spacing:0}td,th{padding:0}/*! Source: https://github.com/h5bp/html5-boilerplate/blob/master/src/css/main.css */@media print{*,*:before,*:after{background:transparent !important;color:#000 !important;-webkit-box-shadow:none !important;box-shadow:none !important;text-shadow:none !important}a,a:visited{text-decoration:underline}a[href]:after{content:" (" attr(href) ")"}abbr[title]:after{content:" (" attr(title) ")"}a[href^="#"]:after,a[href^="javascript:"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}p,h2,h3{orphans:3;widows:3}h2,h3{page-break-after:avoid}.navbar{display:none}.btn>.caret,.dropup>.btn>.caret{border-top-color:#000 !important}.label{border:1px solid #000}.table{border-collapse:collapse !important}.table td,.table th{background-color:#fff !important}.t
able-bordered th,.table-bordered td{border:1px solid #ddd !important}}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}*:before,*:after{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}html{font-size:10px;-webkit-tap-highlight-color:rgba(0,0,0,0)}body{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:1.42857143;color:#333;background-color:#fff}input,button,select,textarea{font-family:inherit;font-size:inherit;line-height:inherit}a{color:#337ab7;text-decoration:none}a:hover,a:focus{color:#23527c;text-decoration:underline}a:focus{outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}figure{margin:0}img{vertical-align:middle}.img-responsive,.thumbnail>img,.thumbnail a>img,.carousel-inner>.item>img,.carousel-inner>.item>a>img{display:block;max-width:100%;height:auto}.img-rounded{border-radius:6px}.img-thumbnail{padding:4px;line-height:1.42857143;background-color:#fff;border:1px solid #
ddd;border-radius:4px;-webkit-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out;display:inline-block;max-width:100%;height:auto}.img-circle{border-radius:50%}hr{margin-top:20px;margin-bottom:20px;border:0;border-top:1px solid #eee}.sr-only{position:absolute;width:1px;height:1px;margin:-1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}[role="button"]{cursor:pointer}h1,h2,h3,h4,h5,h6,.h1,.h2,.h3,.h4,.h5,.h6{font-family:inherit;font-weight:500;line-height:1.1;color:inherit}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small,.h1 small,.h2 small,.h3 small,.h4 small,.h5 small,.h6 small,h1 .small,h2 .small,h3 .small,h4 .small,h5 .small,h6 .small,.h1 .small,.h2 .small,.h3 .small,.h4 .small,.h5 .small,.h6 .small{font-weight:normal;line-height:1;color:#777}h1,.h1,h2,.h2,h3,.h3{margin-top:20px;margin-bottom:10p
x}h1 small,.h1 small,h2 small,.h2 small,h3 small,.h3 small,h1 .small,.h1 .small,h2 .small,.h2 .small,h3 .small,.h3 .small{font-size:65%}h4,.h4,h5,.h5,h6,.h6{margin-top:10px;margin-bottom:10px}h4 small,.h4 small,h5 small,.h5 small,h6 small,.h6 small,h4 .small,.h4 .small,h5 .small,.h5 .small,h6 .small,.h6 .small{font-size:75%}h1,.h1{font-size:36px}h2,.h2{font-size:30px}h3,.h3{font-size:24px}h4,.h4{font-size:18px}h5,.h5{font-size:14px}h6,.h6{font-size:12px}p{margin:0 0 10px}.lead{margin-bottom:20px;font-size:16px;font-weight:300;line-height:1.4}@media (min-width:768px){.lead{font-size:21px}}small,.small{font-size:85%}mark,.mark{background-color:#fcf8e3;padding:.2em}.text-left{text-align:left}.text-right{text-align:right}.text-center{text-align:center}.text-justify{text-align:justify}.text-nowrap{white-space:nowrap}.text-lowercase{text-transform:lowercase}.text-uppercase{text-transform:uppercase}.text-capitalize{text-transform:capitalize}.text-muted{color:#777}.text-primary{color:#337ab
7}a.text-primary:hover,a.text-primary:focus{color:#286090}.text-success{color:#3c763d}a.text-success:hover,a.text-success:focus{color:#2b542c}.text-info{color:#31708f}a.text-info:hover,a.text-info:focus{color:#245269}.text-warning{color:#8a6d3b}a.text-warning:hover,a.text-warning:focus{color:#66512c}.text-danger{color:#a94442}a.text-danger:hover,a.text-danger:focus{color:#843534}.bg-primary{color:#fff;background-color:#337ab7}a.bg-primary:hover,a.bg-primary:focus{background-color:#286090}.bg-success{background-color:#dff0d8}a.bg-success:hover,a.bg-success:focus{background-color:#c1e2b3}.bg-info{background-color:#d9edf7}a.bg-info:hover,a.bg-info:focus{background-color:#afd9ee}.bg-warning{background-color:#fcf8e3}a.bg-warning:hover,a.bg-warning:focus{background-color:#f7ecb5}.bg-danger{background-color:#f2dede}a.bg-danger:hover,a.bg-danger:focus{background-color:#e4b9b9}.page-header{padding-bottom:9px;margin:40px 0 20px;border-bottom:1px solid #eee}ul,ol{margin-top:0;margin-bottom:10p
x}ul ul,ol ul,ul ol,ol ol{margin-bottom:0}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none;margin-left:-5px}.list-inline>li{display:inline-block;padding-left:5px;padding-right:5px}dl{margin-top:0;margin-bottom:20px}dt,dd{line-height:1.42857143}dt{font-weight:bold}dd{margin-left:0}@media (min-width:768px){.dl-horizontal dt{float:left;width:160px;clear:left;text-align:right;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:180px}}abbr[title],abbr[data-original-title]{cursor:help;border-bottom:1px dotted #777}.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:10px 20px;margin:0 0 20px;font-size:17.5px;border-left:5px solid #eee}blockquote p:last-child,blockquote ul:last-child,blockquote ol:last-child{margin-bottom:0}blockquote footer,blockquote small,blockquote .small{display:block;font-size:80%;line-height:1.42857143;color:#777}blockquote footer:before,blockquote small:before,blockquote .
small:before{content:'\2014 \00A0'}.blockquote-reverse,blockquote.pull-right{padding-right:15px;padding-left:0;border-right:5px solid #eee;border-left:0;text-align:right}.blockquote-reverse footer:before,blockquote.pull-right footer:before,.blockquote-reverse small:before,blockquote.pull-right small:before,.blockquote-reverse .small:before,blockquote.pull-right .small:before{content:''}.blockquote-reverse footer:after,blockquote.pull-right footer:after,.blockquote-reverse small:after,blockquote.pull-right small:after,.blockquote-reverse .small:after,blockquote.pull-right .small:after{content:'\00A0 \2014'}address{margin-bottom:20px;font-style:normal;line-height:1.42857143}code,kbd,pre,samp{font-family:Menlo,Monaco,Consolas,"Courier New",monospace}code{padding:2px 4px;font-size:90%;color:#c7254e;background-color:#f9f2f4;border-radius:4px}kbd{padding:2px 4px;font-size:90%;color:#fff;background-color:#333;border-radius:3px;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.25);box-shadow:i
nset 0 -1px 0 rgba(0,0,0,0.25)}kbd kbd{padding:0;font-size:100%;font-weight:bold;-webkit-box-shadow:none;box-shadow:none}pre{display:block;padding:9.5px;margin:0 0 10px;font-size:13px;line-height:1.42857143;word-break:break-all;word-wrap:break-word;color:#333;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px}pre code{padding:0;font-size:inherit;color:inherit;white-space:pre-wrap;background-color:transparent;border-radius:0}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{margin-right:auto;margin-left:auto;padding-left:15px;padding-right:15px}@media (min-width:768px){.container{width:750px}}@media (min-width:992px){.container{width:970px}}@media (min-width:1200px){.container{width:1170px}}.container-fluid{margin-right:auto;margin-left:auto;padding-left:15px;padding-right:15px}.row{margin-left:-15px;margin-right:-15px}.col-xs-1, .col-sm-1, .col-md-1, .col-lg-1, .col-xs-2, .col-sm-2, .col-md-2, .col-lg-2, .col-xs-3, .col-sm-3, .col-md-3, .col-lg-3, .col-xs-4
, .col-sm-4, .col-md-4, .col-lg-4, .col-xs-5, .col-sm-5, .col-md-5, .col-lg-5, .col-xs-6, .col-sm-6, .col-md-6, .col-lg-6, .col-xs-7, .col-sm-7, .col-md-7, .col-lg-7, .col-xs-8, .col-sm-8, .col-md-8, .col-lg-8, .col-xs-9, .col-sm-9, .col-md-9, .col-lg-9, .col-xs-10, .col-sm-10, .col-md-10, .col-lg-10, .col-xs-11, .col-sm-11, .col-md-11, .col-lg-11, .col-xs-12, .col-sm-12, .col-md-12, .col-lg-12{position:relative;min-height:1px;padding-left:15px;padding-right:15px}.col-xs-1, .col-xs-2, .col-xs-3, .col-xs-4, .col-xs-5, .col-xs-6, .col-xs-7, .col-xs-8, .col-xs-9, .col-xs-10, .col-xs-11, .col-xs-12{float:left}.col-xs-12{width:100%}.col-xs-11{width:91.66666667%}.col-xs-10{width:83.33333333%}.col-xs-9{width:75%}.col-xs-8{width:66.66666667%}.col-xs-7{width:58.33333333%}.col-xs-6{width:50%}.col-xs-5{width:41.66666667%}.col-xs-4{width:33.33333333%}.col-xs-3{width:25%}.col-xs-2{width:16.66666667%}.col-xs-1{width:8.33333333%}.col-xs-pull-12{right:100%}.col-xs-pull-11{right:91.66666667%}.col-xs
-pull-10{right:83.33333333%}.col-xs-pull-9{right:75%}.col-xs-pull-8{right:66.66666667%}.col-xs-pull-7{right:58.33333333%}.col-xs-pull-6{right:50%}.col-xs-pull-5{right:41.66666667%}.col-xs-pull-4{right:33.33333333%}.col-xs-pull-3{right:25%}.col-xs-pull-2{right:16.66666667%}.col-xs-pull-1{right:8.33333333%}.col-xs-pull-0{right:auto}.col-xs-push-12{left:100%}.col-xs-push-11{left:91.66666667%}.col-xs-push-10{left:83.33333333%}.col-xs-push-9{left:75%}.col-xs-push-8{left:66.66666667%}.col-xs-push-7{left:58.33333333%}.col-xs-push-6{left:50%}.col-xs-push-5{left:41.66666667%}.col-xs-push-4{left:33.33333333%}.col-xs-push-3{left:25%}.col-xs-push-2{left:16.66666667%}.col-xs-push-1{left:8.33333333%}.col-xs-push-0{left:auto}.col-xs-offset-12{margin-left:100%}.col-xs-offset-11{margin-left:91.66666667%}.col-xs-offset-10{margin-left:83.33333333%}.col-xs-offset-9{margin-left:75%}.col-xs-offset-8{margin-left:66.66666667%}.col-xs-offset-7{margin-left:58.33333333%}.col-xs-offset-6{margin-left:50%}.col-x
s-offset-5{margin-left:41.66666667%}.col-xs-offset-4{margin-left:33.33333333%}.col-xs-offset-3{margin-left:25%}.col-xs-offset-2{margin-left:16.66666667%}.col-xs-offset-1{margin-left:8.33333333%}.col-xs-offset-0{margin-left:0}@media (min-width:768px){.col-sm-1, .col-sm-2, .col-sm-3, .col-sm-4, .col-sm-5, .col-sm-6, .col-sm-7, .col-sm-8, .col-sm-9, .col-sm-10, .col-sm-11, .col-sm-12{float:left}.col-sm-12{width:100%}.col-sm-11{width:91.66666667%}.col-sm-10{width:83.33333333%}.col-sm-9{width:75%}.col-sm-8{width:66.66666667%}.col-sm-7{width:58.33333333%}.col-sm-6{width:50%}.col-sm-5{width:41.66666667%}.col-sm-4{width:33.33333333%}.col-sm-3{width:25%}.col-sm-2{width:16.66666667%}.col-sm-1{width:8.33333333%}.col-sm-pull-12{right:100%}.col-sm-pull-11{right:91.66666667%}.col-sm-pull-10{right:83.33333333%}.col-sm-pull-9{right:75%}.col-sm-pull-8{right:66.66666667%}.col-sm-pull-7{right:58.33333333%}.col-sm-pull-6{right:50%}.col-sm-pull-5{right:41.66666667%}.col-sm-pull-4{right:33.33333333%}.col
-sm-pull-3{right:25%}.col-sm-pull-2{right:16.66666667%}.col-sm-pull-1{right:8.33333333%}.col-sm-pull-0{right:auto}.col-sm-push-12{left:100%}.col-sm-push-11{left:91.66666667%}.col-sm-push-10{left:83.33333333%}.col-sm-push-9{left:75%}.col-sm-push-8{left:66.66666667%}.col-sm-push-7{left:58.33333333%}.col-sm-push-6{left:50%}.col-sm-push-5{left:41.66666667%}.col-sm-push-4{left:33.33333333%}.col-sm-push-3{left:25%}.col-sm-push-2{left:16.66666667%}.col-sm-push-1{left:8.33333333%}.col-sm-push-0{left:auto}.col-sm-offset-12{margin-left:100%}.col-sm-offset-11{margin-left:91.66666667%}.col-sm-offset-10{margin-left:83.33333333%}.col-sm-offset-9{margin-left:75%}.col-sm-offset-8{margin-left:66.66666667%}.col-sm-offset-7{margin-left:58.33333333%}.col-sm-offset-6{margin-left:50%}.col-sm-offset-5{margin-left:41.66666667%}.col-sm-offset-4{margin-left:33.33333333%}.col-sm-offset-3{margin-left:25%}.col-sm-offset-2{margin-left:16.66666667%}.col-sm-offset-1{margin-left:8.33333333%}.col-sm-offset-0{margin-
left:0}}@media (min-width:992px){.col-md-1, .col-md-2, .col-md-3, .col-md-4, .col-md-5, .col-md-6, .col-md-7, .col-md-8, .col-md-9, .col-md-10, .col-md-11, .col-md-12{float:left}.col-md-12{width:100%}.col-md-11{width:91.66666667%}.col-md-10{width:83.33333333%}.col-md-9{width:75%}.col-md-8{width:66.66666667%}.col-md-7{width:58.33333333%}.col-md-6{width:50%}.col-md-5{width:41.66666667%}.col-md-4{width:33.33333333%}.col-md-3{width:25%}.col-md-2{width:16.66666667%}.col-md-1{width:8.33333333%}.col-md-pull-12{right:100%}.col-md-pull-11{right:91.66666667%}.col-md-pull-10{right:83.33333333%}.col-md-pull-9{right:75%}.col-md-pull-8{right:66.66666667%}.col-md-pull-7{right:58.33333333%}.col-md-pull-6{right:50%}.col-md-pull-5{right:41.66666667%}.col-md-pull-4{right:33.33333333%}.col-md-pull-3{right:25%}.col-md-pull-2{right:16.66666667%}.col-md-pull-1{right:8.33333333%}.col-md-pull-0{right:auto}.col-md-push-12{left:100%}.col-md-push-11{left:91.66666667%}.col-md-push-10{left:83.33333333%}.col-md-p
ush-9{left:75%}.col-md-push-8{left:66.66666667%}.col-md-push-7{left:58.33333333%}.col-md-push-6{left:50%}.col-md-push-5{left:41.66666667%}.col-md-push-4{left:33.33333333%}.col-md-push-3{left:25%}.col-md-push-2{left:16.66666667%}.col-md-push-1{left:8.33333333%}.col-md-push-0{left:auto}.col-md-offset-12{margin-left:100%}.col-md-offset-11{margin-left:91.66666667%}.col-md-offset-10{margin-left:83.33333333%}.col-md-offset-9{margin-left:75%}.col-md-offset-8{margin-left:66.66666667%}.col-md-offset-7{margin-left:58.33333333%}.col-md-offset-6{margin-left:50%}.col-md-offset-5{margin-left:41.66666667%}.col-md-offset-4{margin-left:33.33333333%}.col-md-offset-3{margin-left:25%}.col-md-offset-2{margin-left:16.66666667%}.col-md-offset-1{margin-left:8.33333333%}.col-md-offset-0{margin-left:0}}@media (min-width:1200px){.col-lg-1, .col-lg-2, .col-lg-3, .col-lg-4, .col-lg-5, .col-lg-6, .col-lg-7, .col-lg-8, .col-lg-9, .col-lg-10, .col-lg-11, .col-lg-12{float:left}.col-lg-12{width:100%}.col-lg-11{width
:91.66666667%}.col-lg-10{width:83.33333333%}.col-lg-9{width:75%}.col-lg-8{width:66.66666667%}.col-lg-7{width:58.33333333%}.col-lg-6{width:50%}.col-lg-5{width:41.66666667%}.col-lg-4{width:33.33333333%}.col-lg-3{width:25%}.col-lg-2{width:16.66666667%}.col-lg-1{width:8.33333333%}.col-lg-pull-12{right:100%}.col-lg-pull-11{right:91.66666667%}.col-lg-pull-10{right:83.33333333%}.col-lg-pull-9{right:75%}.col-lg-pull-8{right:66.66666667%}.col-lg-pull-7{right:58.33333333%}.col-lg-pull-6{right:50%}.col-lg-pull-5{right:41.66666667%}.col-lg-pull-4{right:33.33333333%}.col-lg-pull-3{right:25%}.col-lg-pull-2{right:16.66666667%}.col-lg-pull-1{right:8.33333333%}.col-lg-pull-0{right:auto}.col-lg-push-12{left:100%}.col-lg-push-11{left:91.66666667%}.col-lg-push-10{left:83.33333333%}.col-lg-push-9{left:75%}.col-lg-push-8{left:66.66666667%}.col-lg-push-7{left:58.33333333%}.col-lg-push-6{left:50%}.col-lg-push-5{left:41.66666667%}.col-lg-push-4{left:33.33333333%}.col-lg-push-3{left:25%}.col-lg-push-2{left:1
6.66666667%}.col-lg-push-1{left:8.33333333%}.col-lg-push-0{left:auto}.col-lg-offset-12{margin-left:100%}.col-lg-offset-11{margin-left:91.66666667%}.col-lg-offset-10{margin-left:83.33333333%}.col-lg-offset-9{margin-left:75%}.col-lg-offset-8{margin-left:66.66666667%}.col-lg-offset-7{margin-left:58.33333333%}.col-lg-offset-6{margin-left:50%}.col-lg-offset-5{margin-left:41.66666667%}.col-lg-offset-4{margin-left:33.33333333%}.col-lg-offset-3{margin-left:25%}.col-lg-offset-2{margin-left:16.66666667%}.col-lg-offset-1{margin-left:8.33333333%}.col-lg-offset-0{margin-left:0}}table{background-color:transparent}caption{padding-top:8px;padding-bottom:8px;color:#777;text-align:left}th{text-align:left}.table{width:100%;max-width:100%;margin-bottom:20px}.table>thead>tr>th,.table>tbody>tr>th,.table>tfoot>tr>th,.table>thead>tr>td,.table>tbody>tr>td,.table>tfoot>tr>td{padding:8px;line-height:1.42857143;vertical-align:top;border-top:1px solid #ddd}.table>thead>tr>th{vertical-align:bottom;border-bottom:
2px solid #ddd}.table>caption+thead>tr:first-child>th,.table>colgroup+thead>tr:first-child>th,.table>thead:first-child>tr:first-child>th,.table>caption+thead>tr:first-child>td,.table>colgroup+thead>tr:first-child>td,.table>thead:first-child>tr:first-child>td{border-top:0}.table>tbody+tbody{border-top:2px solid #ddd}.table .table{background-color:#fff}.table-condensed>thead>tr>th,.table-condensed>tbody>tr>th,.table-condensed>tfoot>tr>th,.table-condensed>thead>tr>td,.table-condensed>tbody>tr>td,.table-condensed>tfoot>tr>td{padding:5px}.table-bordered{border:1px solid #ddd}.table-bordered>thead>tr>th,.table-bordered>tbody>tr>th,.table-bordered>tfoot>tr>th,.table-bordered>thead>tr>td,.table-bordered>tbody>tr>td,.table-bordered>tfoot>tr>td{border:1px solid #ddd}.table-bordered>thead>tr>th,.table-bordered>thead>tr>td{border-bottom-width:2px}.table-striped>tbody>tr:nth-of-type(odd){background-color:#f9f9f9}.table-hover>tbody>tr:hover{background-color:#f5f5f5}table col[class*="col-"]{positi
on:static;float:none;display:table-column}table td[class*="col-"],table th[class*="col-"]{position:static;float:none;display:table-cell}.table>thead>tr>td.active,.table>tbody>tr>td.active,.table>tfoot>tr>td.active,.table>thead>tr>th.active,.table>tbody>tr>th.active,.table>tfoot>tr>th.active,.table>thead>tr.active>td,.table>tbody>tr.active>td,.table>tfoot>tr.active>td,.table>thead>tr.active>th,.table>tbody>tr.active>th,.table>tfoot>tr.active>th{background-color:#f5f5f5}.table-hover>tbody>tr>td.active:hover,.table-hover>tbody>tr>th.active:hover,.table-hover>tbody>tr.active:hover>td,.table-hover>tbody>tr:hover>.active,.table-hover>tbody>tr.active:hover>th{background-color:#e8e8e8}.table>thead>tr>td.success,.table>tbody>tr>td.success,.table>tfoot>tr>td.success,.table>thead>tr>th.success,.table>tbody>tr>th.success,.table>tfoot>tr>th.success,.table>thead>tr.success>td,.table>tbody>tr.success>td,.table>tfoot>tr.success>td,.table>thead>tr.success>th,.table>tbody>tr.success>th,.table>tfoot>t
r.success>th{background-color:#dff0d8}.table-hover>tbody>tr>td.success:hover,.table-hover>tbody>tr>th.success:hover,.table-hover>tbody>tr.success:hover>td,.table-hover>tbody>tr:hover>.success,.table-hover>tbody>tr.success:hover>th{background-color:#d0e9c6}.table>thead>tr>td.info,.table>tbody>tr>td.info,.table>tfoot>tr>td.info,.table>thead>tr>th.info,.table>tbody>tr>th.info,.table>tfoot>tr>th.info,.table>thead>tr.info>td,.table>tbody>tr.info>td,.table>tfoot>tr.info>td,.table>thead>tr.info>th,.table>tbody>tr.info>th,.table>tfoot>tr.info>th{background-color:#d9edf7}.table-hover>tbody>tr>td.info:hover,.table-hover>tbody>tr>th.info:hover,.table-hover>tbody>tr.info:hover>td,.table-hover>tbody>tr:hover>.info,.table-hover>tbody>tr.info:hover>th{background-color:#c4e3f3}.table>thead>tr>td.warning,.table>tbody>tr>td.warning,.table>tfoot>tr>td.warning,.table>thead>tr>th.warning,.table>tbody>tr>th.warning,.table>tfoot>tr>th.warning,.table>thead>tr.warning>td,.table>tbody>tr.warning>td,.table>tf
oot>tr.warning>td,.table>thead>tr.warning>th,.table>tbody>tr.warning>th,.table>tfoot>tr.warning>th{background-color:#fcf8e3}.table-hover>tbody>tr>td.warning:hover,.table-hover>tbody>tr>th.warning:hover,.table-hover>tbody>tr.warning:hover>td,.table-hover>tbody>tr:hover>.warning,.table-hover>tbody>tr.warning:hover>th{background-color:#faf2cc}.table>thead>tr>td.danger,.table>tbody>tr>td.danger,.table>tfoot>tr>td.danger,.table>thead>tr>th.danger,.table>tbody>tr>th.danger,.table>tfoot>tr>th.danger,.table>thead>tr.danger>td,.table>tbody>tr.danger>td,.table>tfoot>tr.danger>td,.table>thead>tr.danger>th,.table>tbody>tr.danger>th,.table>tfoot>tr.danger>th{background-color:#f2dede}.table-hover>tbody>tr>td.danger:hover,.table-hover>tbody>tr>th.danger:hover,.table-hover>tbody>tr.danger:hover>td,.table-hover>tbody>tr:hover>.danger,.table-hover>tbody>tr.danger:hover>th{background-color:#ebcccc}.table-responsive{overflow-x:auto;min-height:0.01%}@media screen and (max-width:767px){.table-responsive{
width:100%;margin-bottom:15px;overflow-y:hidden;-ms-overflow-style:-ms-autohiding-scrollbar;border:1px solid #ddd}.table-responsive>.table{margin-bottom:0}.table-responsive>.table>thead>tr>th,.table-responsive>.table>tbody>tr>th,.table-responsive>.table>tfoot>tr>th,.table-responsive>.table>thead>tr>td,.table-responsive>.table>tbody>tr>td,.table-responsive>.table>tfoot>tr>td{white-space:nowrap}.table-responsive>.table-bordered{border:0}.table-responsive>.table-bordered>thead>tr>th:first-child,.table-responsive>.table-bordered>tbody>tr>th:first-child,.table-responsive>.table-bordered>tfoot>tr>th:first-child,.table-responsive>.table-bordered>thead>tr>td:first-child,.table-responsive>.table-bordered>tbody>tr>td:first-child,.table-responsive>.table-bordered>tfoot>tr>td:first-child{border-left:0}.table-responsive>.table-bordered>thead>tr>th:last-child,.table-responsive>.table-bordered>tbody>tr>th:last-child,.table-responsive>.table-bordered>tfoot>tr>th:last-child,.table-responsive>.table-
bordered>thead>tr>td:last-child,.table-responsive>.table-bordered>tbody>tr>td:last-child,.table-responsive>.table-bordered>tfoot>tr>td:last-child{border-right:0}.table-responsive>.table-bordered>tbody>tr:last-child>th,.table-responsive>.table-bordered>tfoot>tr:last-child>th,.table-responsive>.table-bordered>tbody>tr:last-child>td,.table-responsive>.table-bordered>tfoot>tr:last-child>td{border-bottom:0}}fieldset{padding:0;margin:0;border:0;min-width:0}legend{display:block;width:100%;padding:0;margin-bottom:20px;font-size:21px;line-height:inherit;color:#333;border:0;border-bottom:1px solid #e5e5e5}label{display:inline-block;max-width:100%;margin-bottom:5px;font-weight:bold}input[type="search"]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type="radio"],input[type="checkbox"]{margin:4px 0 0;margin-top:1px \9;line-height:normal}input[type="file"]{display:block}input[type="range"]{display:block;width:100%}select[multiple],select[size]{height:auto}in
put[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus{outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}output{display:block;padding-top:7px;font-size:14px;line-height:1.42857143;color:#555}.form-control{display:block;width:100%;height:34px;padding:6px 12px;font-size:14px;line-height:1.42857143;color:#555;background-color:#fff;background-image:none;border:1px solid #ccc;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-webkit-transition:border-color ease-in-out .15s, -webkit-box-shadow ease-in-out .15s;-o-transition:border-color ease-in-out .15s, box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s, box-shadow ease-in-out .15s}.form-control:focus{border-color:#66afe9;outline:0;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6);box-shadow:inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6)}.form-control::-moz-placehol
der{color:#999;opacity:1}.form-control:-ms-input-placeholder{color:#999}.form-control::-webkit-input-placeholder{color:#999}.form-control::-ms-expand{border:0;background-color:transparent}.form-control[disabled],.form-control[readonly],fieldset[disabled] .form-control{background-color:#eee;opacity:1}.form-control[disabled],fieldset[disabled] .form-control{cursor:not-allowed}textarea.form-control{height:auto}input[type="search"]{-webkit-appearance:none}@media screen and (-webkit-min-device-pixel-ratio:0){input[type="date"].form-control,input[type="time"].form-control,input[type="datetime-local"].form-control,input[type="month"].form-control{line-height:34px}input[type="date"].input-sm,input[type="time"].input-sm,input[type="datetime-local"].input-sm,input[type="month"].input-sm,.input-group-sm input[type="date"],.input-group-sm input[type="time"],.input-group-sm input[type="datetime-local"],.input-group-sm input[type="month"]{line-height:30px}input[type="date"].input-lg,input[type="t
ime"].input-lg,input[type="datetime-local"].input-lg,input[type="month"].input-lg,.input-group-lg input[type="date"],.input-group-lg input[type="time"],.input-group-lg input[type="datetime-local"],.input-group-lg input[type="month"]{line-height:46px}}.form-group{margin-bottom:15px}.radio,.checkbox{position:relative;display:block;margin-top:10px;margin-bottom:10px}.radio label,.checkbox label{min-height:20px;padding-left:20px;margin-bottom:0;font-weight:normal;cursor:pointer}.radio input[type="radio"],.radio-inline input[type="radio"],.checkbox input[type="checkbox"],.checkbox-inline input[type="checkbox"]{position:absolute;margin-left:-20px;margin-top:4px \9}.radio+.radio,.checkbox+.checkbox{margin-top:-5px}.radio-inline,.checkbox-inline{position:relative;display:inline-block;padding-left:20px;margin-bottom:0;vertical-align:middle;font-weight:normal;cursor:pointer}.radio-inline+.radio-inline,.checkbox-inline+.checkbox-inline{margin-top:0;margin-left:10px}input[type="radio"][disabled
],input[type="checkbox"][disabled],input[type="radio"].disabled,input[type="checkbox"].disabled,fieldset[disabled] input[type="radio"],fieldset[disabled] input[type="checkbox"]{cursor:not-allowed}.radio-inline.disabled,.checkbox-inline.disabled,fieldset[disabled] .radio-inline,fieldset[disabled] .checkbox-inline{cursor:not-allowed}.radio.disabled label,.checkbox.disabled label,fieldset[disabled] .radio label,fieldset[disabled] .checkbox label{cursor:not-allowed}.form-control-static{padding-top:7px;padding-bottom:7px;margin-bottom:0;min-height:34px}.form-control-static.input-lg,.form-control-static.input-sm{padding-left:0;padding-right:0}.input-sm{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}select.input-sm{height:30px;line-height:30px}textarea.input-sm,select[multiple].input-sm{height:auto}.form-group-sm .form-control{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.form-group-sm select.form-control{height:30px;line-heig
ht:30px}.form-group-sm textarea.form-control,.form-group-sm select[multiple].form-control{height:auto}.form-group-sm .form-control-static{height:30px;min-height:32px;padding:6px 10px;font-size:12px;line-height:1.5}.input-lg{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}select.input-lg{height:46px;line-height:46px}textarea.input-lg,select[multiple].input-lg{height:auto}.form-group-lg .form-control{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.form-group-lg select.form-control{height:46px;line-height:46px}.form-group-lg textarea.form-control,.form-group-lg select[multiple].form-control{height:auto}.form-group-lg .form-control-static{height:46px;min-height:38px;padding:11px 16px;font-size:18px;line-height:1.3333333}.has-feedback{position:relative}.has-feedback .form-control{padding-right:42.5px}.form-control-feedback{position:absolute;top:0;right:0;z-index:2;display:block;width:34px;height:34px;line-height:3
4px;text-align:center;pointer-events:none}.input-lg+.form-control-feedback,.input-group-lg+.form-control-feedback,.form-group-lg .form-control+.form-control-feedback{width:46px;height:46px;line-height:46px}.input-sm+.form-control-feedback,.input-group-sm+.form-control-feedback,.form-group-sm .form-control+.form-control-feedback{width:30px;height:30px;line-height:30px}.has-success .help-block,.has-success .control-label,.has-success .radio,.has-success .checkbox,.has-success .radio-inline,.has-success .checkbox-inline,.has-success.radio label,.has-success.checkbox label,.has-success.radio-inline label,.has-success.checkbox-inline label{color:#3c763d}.has-success .form-control{border-color:#3c763d;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.has-success .form-control:focus{border-color:#2b542c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #67b168;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #67b168}.has-succe
ss .input-group-addon{color:#3c763d;border-color:#3c763d;background-color:#dff0d8}.has-success .form-control-feedback{color:#3c763d}.has-warning .help-block,.has-warning .control-label,.has-warning .radio,.has-warning .checkbox,.has-warning .radio-inline,.has-warning .checkbox-inline,.has-warning.radio label,.has-warning.checkbox label,.has-warning.radio-inline label,.has-warning.checkbox-inline label{color:#8a6d3b}.has-warning .form-control{border-color:#8a6d3b;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.has-warning .form-control:focus{border-color:#66512c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #c0a16b;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #c0a16b}.has-warning .input-group-addon{color:#8a6d3b;border-color:#8a6d3b;background-color:#fcf8e3}.has-warning .form-control-feedback{color:#8a6d3b}.has-error .help-block,.has-error .control-label,.has-error .radio,.has-error .checkbox,.has-error .radio-
inline,.has-error .checkbox-inline,.has-error.radio label,.has-error.checkbox label,.has-error.radio-inline label,.has-error.checkbox-inline label{color:#a94442}.has-error .form-control{border-color:#a94442;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.has-error .form-control:focus{border-color:#843534;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #ce8483;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #ce8483}.has-error .input-group-addon{color:#a94442;border-color:#a94442;background-color:#f2dede}.has-error .form-control-feedback{color:#a94442}.has-feedback label~.form-control-feedback{top:25px}.has-feedback label.sr-only~.form-control-feedback{top:0}.help-block{display:block;margin-top:5px;margin-bottom:10px;color:#737373}@media (min-width:768px){.form-inline .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle
}.form-inline .form-control-static{display:inline-block}.form-inline .input-group{display:inline-table;vertical-align:middle}.form-inline .input-group .input-group-addon,.form-inline .input-group .input-group-btn,.form-inline .input-group .form-control{width:auto}.form-inline .input-group>.form-control{width:100%}.form-inline .control-label{margin-bottom:0;vertical-align:middle}.form-inline .radio,.form-inline .checkbox{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle}.form-inline .radio label,.form-inline .checkbox label{padding-left:0}.form-inline .radio input[type="radio"],.form-inline .checkbox input[type="checkbox"]{position:relative;margin-left:0}.form-inline .has-feedback .form-control-feedback{top:0}}.form-horizontal .radio,.form-horizontal .checkbox,.form-horizontal .radio-inline,.form-horizontal .checkbox-inline{margin-top:0;margin-bottom:0;padding-top:7px}.form-horizontal .radio,.form-horizontal .checkbox{min-height:27px}.form-horizontal .form-group
{margin-left:-15px;margin-right:-15px}@media (min-width:768px){.form-horizontal .control-label{text-align:right;margin-bottom:0;padding-top:7px}}.form-horizontal .has-feedback .form-control-feedback{right:15px}@media (min-width:768px){.form-horizontal .form-group-lg .control-label{padding-top:11px;font-size:18px}}@media (min-width:768px){.form-horizontal .form-group-sm .control-label{padding-top:6px;font-size:12px}}.btn{display:inline-block;margin-bottom:0;font-weight:normal;text-align:center;vertical-align:middle;-ms-touch-action:manipulation;touch-action:manipulation;cursor:pointer;background-image:none;border:1px solid transparent;white-space:nowrap;padding:6px 12px;font-size:14px;line-height:1.42857143;border-radius:4px;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.btn:focus,.btn:active:focus,.btn.active:focus,.btn.focus,.btn:active.focus,.btn.active.focus{outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn:hover,.btn:focus,
.btn.focus{color:#333;text-decoration:none}.btn:active,.btn.active{outline:0;background-image:none;-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,0.125);box-shadow:inset 0 3px 5px rgba(0,0,0,0.125)}.btn.disabled,.btn[disabled],fieldset[disabled] .btn{cursor:not-allowed;opacity:.65;filter:alpha(opacity=65);-webkit-box-shadow:none;box-shadow:none}a.btn.disabled,fieldset[disabled] a.btn{pointer-events:none}.btn-default{color:#333;background-color:#fff;border-color:#ccc}.btn-default:focus,.btn-default.focus{color:#333;background-color:#e6e6e6;border-color:#8c8c8c}.btn-default:hover{color:#333;background-color:#e6e6e6;border-color:#adadad}.btn-default:active,.btn-default.active,.open>.dropdown-toggle.btn-default{color:#333;background-color:#e6e6e6;border-color:#adadad}.btn-default:active:hover,.btn-default.active:hover,.open>.dropdown-toggle.btn-default:hover,.btn-default:active:focus,.btn-default.active:focus,.open>.dropdown-toggle.btn-default:focus,.btn-default:active.focus,.btn-default
.active.focus,.open>.dropdown-toggle.btn-default.focus{color:#333;background-color:#d4d4d4;border-color:#8c8c8c}.btn-default:active,.btn-default.active,.open>.dropdown-toggle.btn-default{background-image:none}.btn-default.disabled:hover,.btn-default[disabled]:hover,fieldset[disabled] .btn-default:hover,.btn-default.disabled:focus,.btn-default[disabled]:focus,fieldset[disabled] .btn-default:focus,.btn-default.disabled.focus,.btn-default[disabled].focus,fieldset[disabled] .btn-default.focus{background-color:#fff;border-color:#ccc}.btn-default .badge{color:#fff;background-color:#333}.btn-primary{color:#fff;background-color:#337ab7;border-color:#2e6da4}.btn-primary:focus,.btn-primary.focus{color:#fff;background-color:#286090;border-color:#122b40}.btn-primary:hover{color:#fff;background-color:#286090;border-color:#204d74}.btn-primary:active,.btn-primary.active,.open>.dropdown-toggle.btn-primary{color:#fff;background-color:#286090;border-color:#204d74}.btn-primary:active:hover,.btn-primar
y.active:hover,.open>.dropdown-toggle.btn-primary:hover,.btn-primary:active:focus,.btn-primary.active:focus,.open>.dropdown-toggle.btn-primary:focus,.btn-primary:active.focus,.btn-primary.active.focus,.open>.dropdown-toggle.btn-primary.focus{color:#fff;background-color:#204d74;border-color:#122b40}.btn-primary:active,.btn-primary.active,.open>.dropdown-toggle.btn-primary{background-image:none}.btn-primary.disabled:hover,.btn-primary[disabled]:hover,fieldset[disabled] .btn-primary:hover,.btn-primary.disabled:focus,.btn-primary[disabled]:focus,fieldset[disabled] .btn-primary:focus,.btn-primary.disabled.focus,.btn-primary[disabled].focus,fieldset[disabled] .btn-primary.focus{background-color:#337ab7;border-color:#2e6da4}.btn-primary .badge{color:#337ab7;background-color:#fff}.btn-success{color:#fff;background-color:#5cb85c;border-color:#4cae4c}.btn-success:focus,.btn-success.focus{color:#fff;background-color:#449d44;border-color:#255625}.btn-success:hover{color:#fff;background-color:#4
49d44;border-color:#398439}.btn-success:active,.btn-success.active,.open>.dropdown-toggle.btn-success{color:#fff;background-color:#449d44;border-color:#398439}.btn-success:active:hover,.btn-success.active:hover,.open>.dropdown-toggle.btn-success:hover,.btn-success:active:focus,.btn-success.active:focus,.open>.dropdown-toggle.btn-success:focus,.btn-success:active.focus,.btn-success.active.focus,.open>.dropdown-toggle.btn-success.focus{color:#fff;background-color:#398439;border-color:#255625}.btn-success:active,.btn-success.active,.open>.dropdown-toggle.btn-success{background-image:none}.btn-success.disabled:hover,.btn-success[disabled]:hover,fieldset[disabled] .btn-success:hover,.btn-success.disabled:focus,.btn-success[disabled]:focus,fieldset[disabled] .btn-success:focus,.btn-success.disabled.focus,.btn-success[disabled].focus,fieldset[disabled] .btn-success.focus{background-color:#5cb85c;border-color:#4cae4c}.btn-success .badge{color:#5cb85c;background-color:#fff}.btn-info{color:#f
ff;background-color:#5bc0de;border-color:#46b8da}.btn-info:focus,.btn-info.focus{color:#fff;background-color:#31b0d5;border-color:#1b6d85}.btn-info:hover{color:#fff;background-color:#31b0d5;border-color:#269abc}.btn-info:active,.btn-info.active,.open>.dropdown-toggle.btn-info{color:#fff;background-color:#31b0d5;border-color:#269abc}.btn-info:active:hover,.btn-info.active:hover,.open>.dropdown-toggle.btn-info:hover,.btn-info:active:focus,.btn-info.active:focus,.open>.dropdown-toggle.btn-info:focus,.btn-info:active.focus,.btn-info.active.focus,.open>.dropdown-toggle.btn-info.focus{color:#fff;background-color:#269abc;border-color:#1b6d85}.btn-info:active,.btn-info.active,.open>.dropdown-toggle.btn-info{background-image:none}.btn-info.disabled:hover,.btn-info[disabled]:hover,fieldset[disabled] .btn-info:hover,.btn-info.disabled:focus,.btn-info[disabled]:focus,fieldset[disabled] .btn-info:focus,.btn-info.disabled.focus,.btn-info[disabled].focus,fieldset[disabled] .btn-info.focus{backgrou
nd-color:#5bc0de;border-color:#46b8da}.btn-info .badge{color:#5bc0de;background-color:#fff}.btn-warning{color:#fff;background-color:#f0ad4e;border-color:#eea236}.btn-warning:focus,.btn-warning.focus{color:#fff;background-color:#ec971f;border-color:#985f0d}.btn-warning:hover{color:#fff;background-color:#ec971f;border-color:#d58512}.btn-warning:active,.btn-warning.active,.open>.dropdown-toggle.btn-warning{color:#fff;background-color:#ec971f;border-color:#d58512}.btn-warning:active:hover,.btn-warning.active:hover,.open>.dropdown-toggle.btn-warning:hover,.btn-warning:active:focus,.btn-warning.active:focus,.open>.dropdown-toggle.btn-warning:focus,.btn-warning:active.focus,.btn-warning.active.focus,.open>.dropdown-toggle.btn-warning.focus{color:#fff;background-color:#d58512;border-color:#985f0d}.btn-warning:active,.btn-warning.active,.open>.dropdown-toggle.btn-warning{background-image:none}.btn-warning.disabled:hover,.btn-warning[disabled]:hover,fieldset[disabled] .btn-warning:hover,.btn-
warning.disabled:focus,.btn-warning[disabled]:focus,fieldset[disabled] .btn-warning:focus,.btn-warning.disabled.focus,.btn-warning[disabled].focus,fieldset[disabled] .btn-warning.focus{background-color:#f0ad4e;border-color:#eea236}.btn-warning .badge{color:#f0ad4e;background-color:#fff}.btn-danger{color:#fff;background-color:#d9534f;border-color:#d43f3a}.btn-danger:focus,.btn-danger.focus{color:#fff;background-color:#c9302c;border-color:#761c19}.btn-danger:hover{color:#fff;background-color:#c9302c;border-color:#ac2925}.btn-danger:active,.btn-danger.active,.open>.dropdown-toggle.btn-danger{color:#fff;background-color:#c9302c;border-color:#ac2925}.btn-danger:active:hover,.btn-danger.active:hover,.open>.dropdown-toggle.btn-danger:hover,.btn-danger:active:focus,.btn-danger.active:focus,.open>.dropdown-toggle.btn-danger:focus,.btn-danger:active.focus,.btn-danger.active.focus,.open>.dropdown-toggle.btn-danger.focus{color:#fff;background-color:#ac2925;border-color:#761c19}.btn-danger:activ
e,.btn-danger.active,.open>.dropdown-toggle.btn-danger{background-image:none}.btn-danger.disabled:hover,.btn-danger[disabled]:hover,fieldset[disabled] .btn-danger:hover,.btn-danger.disabled:focus,.btn-danger[disabled]:focus,fieldset[disabled] .btn-danger:focus,.btn-danger.disabled.focus,.btn-danger[disabled].focus,fieldset[disabled] .btn-danger.focus{background-color:#d9534f;border-color:#d43f3a}.btn-danger .badge{color:#d9534f;background-color:#fff}.btn-link{color:#337ab7;font-weight:normal;border-radius:0}.btn-link,.btn-link:active,.btn-link.active,.btn-link[disabled],fieldset[disabled] .btn-link{background-color:transparent;-webkit-box-shadow:none;box-shadow:none}.btn-link,.btn-link:hover,.btn-link:focus,.btn-link:active{border-color:transparent}.btn-link:hover,.btn-link:focus{color:#23527c;text-decoration:underline;background-color:transparent}.btn-link[disabled]:hover,fieldset[disabled] .btn-link:hover,.btn-link[disabled]:focus,fieldset[disabled] .btn-link:focus{color:#777;text
-decoration:none}.btn-lg,.btn-group-lg>.btn{padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.btn-sm,.btn-group-sm>.btn{padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.btn-xs,.btn-group-xs>.btn{padding:1px 5px;font-size:12px;line-height:1.5;border-radius:3px}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:5px}input[type="submit"].btn-block,input[type="reset"].btn-block,input[type="button"].btn-block{width:100%}.fade{opacity:0;-webkit-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1}.collapse{display:none}.collapse.in{display:block}tr.collapse.in{display:table-row}tbody.collapse.in{display:table-row-group}.collapsing{position:relative;height:0;overflow:hidden;-webkit-transition-property:height, visibility;-o-transition-property:height, visibility;transition-property:height, visibility;-webkit-transition-duration:.35s;-o-transition-duration:.35s;transition-
duration:.35s;-webkit-transition-timing-function:ease;-o-transition-timing-function:ease;transition-timing-function:ease}.caret{display:inline-block;width:0;height:0;margin-left:2px;vertical-align:middle;border-top:4px dashed;border-top:4px solid \9;border-right:4px solid transparent;border-left:4px solid transparent}.dropup,.dropdown{position:relative}.dropdown-toggle:focus{outline:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;list-style:none;font-size:14px;text-align:left;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.15);border-radius:4px;-webkit-box-shadow:0 6px 12px rgba(0,0,0,0.175);box-shadow:0 6px 12px rgba(0,0,0,0.175);-webkit-background-clip:padding-box;background-clip:padding-box}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5}.dropdown-menu>li>a{display:block;padding:3px 20px
;clear:both;font-weight:normal;line-height:1.42857143;color:#333;white-space:nowrap}.dropdown-menu>li>a:hover,.dropdown-menu>li>a:focus{text-decoration:none;color:#262626;background-color:#f5f5f5}.dropdown-menu>.active>a,.dropdown-menu>.active>a:hover,.dropdown-menu>.active>a:focus{color:#fff;text-decoration:none;outline:0;background-color:#337ab7}.dropdown-menu>.disabled>a,.dropdown-menu>.disabled>a:hover,.dropdown-menu>.disabled>a:focus{color:#777}.dropdown-menu>.disabled>a:hover,.dropdown-menu>.disabled>a:focus{text-decoration:none;background-color:transparent;background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);cursor:not-allowed}.open>.dropdown-menu{display:block}.open>a{outline:0}.dropdown-menu-right{left:auto;right:0}.dropdown-menu-left{left:0;right:auto}.dropdown-header{display:block;padding:3px 20px;font-size:12px;line-height:1.42857143;color:#777;white-space:nowrap}.dropdown-backdrop{position:fixed;left:0;right:0;bottom:0;top:0;z-index:9
90}.pull-right>.dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0;border-bottom:4px dashed;border-bottom:4px solid \9;content:""}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:2px}@media (min-width:768px){.navbar-right .dropdown-menu{left:auto;right:0}.navbar-right .dropdown-menu-left{left:0;right:auto}}.btn-group,.btn-group-vertical{position:relative;display:inline-block;vertical-align:middle}.btn-group>.btn,.btn-group-vertical>.btn{position:relative;float:left}.btn-group>.btn:hover,.btn-group-vertical>.btn:hover,.btn-group>.btn:focus,.btn-group-vertical>.btn:focus,.btn-group>.btn:active,.btn-group-vertical>.btn:active,.btn-group>.btn.active,.btn-group-vertical>.btn.active{z-index:2}.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group{margin-left:-1px}.btn-toolbar{margin-left:-5px}.btn-toolbar .btn,.btn-toolbar .btn-group,.b
tn-toolbar .input-group{float:left}.btn-toolbar>.btn,.btn-toolbar>.btn-group,.btn-toolbar>.input-group{margin-left:5px}.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle){border-radius:0}.btn-group>.btn:first-child{margin-left:0}.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-top-right-radius:0}.btn-group>.btn:last-child:not(:first-child),.btn-group>.dropdown-toggle:not(:first-child){border-bottom-left-radius:0;border-top-left-radius:0}.btn-group>.btn-group{float:left}.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-top-right-radius:0}.btn-group>.btn-group:last-child:not(:first-child)>.btn:first-child{border-bottom-left-radius:0;border-top-left-radius:0}.btn-group .dropdown-toggle:active,.btn-group.open .dropdow
n-toggle{outline:0}.btn-group>.btn+.dropdown-toggle{padding-left:8px;padding-right:8px}.btn-group>.btn-lg+.dropdown-toggle{padding-left:12px;padding-right:12px}.btn-group.open .dropdown-toggle{-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,0.125);box-shadow:inset 0 3px 5px rgba(0,0,0,0.125)}.btn-group.open .dropdown-toggle.btn-link{-webkit-box-shadow:none;box-shadow:none}.btn .caret{margin-left:0}.btn-lg .caret{border-width:5px 5px 0;border-bottom-width:0}.dropup .btn-lg .caret{border-width:0 5px 5px}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group,.btn-group-vertical>.btn-group>.btn{display:block;float:none;width:100%;max-width:100%}.btn-group-vertical>.btn-group>.btn{float:none}.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0}.btn-group-vertical>.btn:not(:first-child):not(:last-child){border-radius:0}.btn-group-vertical>.btn:first-child:not(:last-child
){border-top-right-radius:4px;border-top-left-radius:4px;border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:last-child:not(:first-child){border-top-right-radius:0;border-top-left-radius:0;border-bottom-right-radius:4px;border-bottom-left-radius:4px}.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-right-radius:0;border-top-left-radius:0}.btn-group-justified{display:table;width:100%;table-layout:fixed;border-collapse:separate}.btn-group-justified>.btn,.btn-group-justified>.btn-group{float:none;display:table-cell;width:1%}.btn-group-justified>.btn-group .btn{width:100%}.btn-group-justified>.btn-group .dropdown-menu{left:auto}
[data-toggle="buttons"]>.btn input[type="radio"],[data-toggle="buttons"]>.btn-group>.btn input[type="radio"],[data-toggle="buttons"]>.btn input[type="checkbox"],[data-toggle="buttons"]>.btn-group>.btn input[type="checkbox"]{position:absolute;clip:rect(0, 0, 0, 0);pointer-events:none}.input-group{position:relative;display:table;border-collapse:separate}.input-group[class*="col-"]{float:none;padding-left:0;padding-right:0}.input-group .form-control{position:relative;z-index:2;float:left;width:100%;margin-bottom:0}.input-group .form-control:focus{z-index:3}.input-group-lg>.form-control,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.btn{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}select.input-group-lg>.form-control,select.input-group-lg>.input-group-addon,select.input-group-lg>.input-group-btn>.btn{height:46px;line-height:46px}textarea.input-group-lg>.form-control,textarea.input-group-lg>.input-group-addon,textarea.input-gro
up-lg>.input-group-btn>.btn,select[multiple].input-group-lg>.form-control,select[multiple].input-group-lg>.input-group-addon,select[multiple].input-group-lg>.input-group-btn>.btn{height:auto}.input-group-sm>.form-control,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.btn{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}select.input-group-sm>.form-control,select.input-group-sm>.input-group-addon,select.input-group-sm>.input-group-btn>.btn{height:30px;line-height:30px}textarea.input-group-sm>.form-control,textarea.input-group-sm>.input-group-addon,textarea.input-group-sm>.input-group-btn>.btn,select[multiple].input-group-sm>.form-control,select[multiple].input-group-sm>.input-group-addon,select[multiple].input-group-sm>.input-group-btn>.btn{height:auto}.input-group-addon,.input-group-btn,.input-group .form-control{display:table-cell}.input-group-addon:not(:first-child):not(:last-child),.input-group-btn:not(:first-child):not(:last-chil
d),.input-group .form-control:not(:first-child):not(:last-child){border-radius:0}.input-group-addon,.input-group-btn{width:1%;white-space:nowrap;vertical-align:middle}.input-group-addon{padding:6px 12px;font-size:14px;font-weight:normal;line-height:1;color:#555;text-align:center;background-color:#eee;border:1px solid #ccc;border-radius:4px}.input-group-addon.input-sm{padding:5px 10px;font-size:12px;border-radius:3px}.input-group-addon.input-lg{padding:10px 16px;font-size:18px;border-radius:6px}.input-group-addon input[type="radio"],.input-group-addon input[type="checkbox"]{margin-top:0}.input-group .form-control:first-child,.input-group-addon:first-child,.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group>.btn,.input-group-btn:first-child>.dropdown-toggle,.input-group-btn:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group-btn:last-child>.btn-group:not(:last-child)>.btn{border-bottom-right-radius:0;border-top-right-radius:0}.input-group-addon:fi
rst-child{border-right:0}.input-group .form-control:last-child,.input-group-addon:last-child,.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group>.btn,.input-group-btn:last-child>.dropdown-toggle,.input-group-btn:first-child>.btn:not(:first-child),.input-group-btn:first-child>.btn-group:not(:first-child)>.btn{border-bottom-left-radius:0;border-top-left-radius:0}.input-group-addon:last-child{border-left:0}.input-group-btn{position:relative;font-size:0;white-space:nowrap}.input-group-btn>.btn{position:relative}.input-group-btn>.btn+.btn{margin-left:-1px}.input-group-btn>.btn:hover,.input-group-btn>.btn:focus,.input-group-btn>.btn:active{z-index:2}.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group{margin-right:-1px}.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group{z-index:2;margin-left:-1px}.nav{margin-bottom:0;padding-left:0;list-style:none}.nav>li{position:relative;display:block}.nav>li>a{position:relative;display:block;
padding:10px 15px}.nav>li>a:hover,.nav>li>a:focus{text-decoration:none;background-color:#eee}.nav>li.disabled>a{color:#777}.nav>li.disabled>a:hover,.nav>li.disabled>a:focus{color:#777;text-decoration:none;background-color:transparent;cursor:not-allowed}.nav .open>a,.nav .open>a:hover,.nav .open>a:focus{background-color:#eee;border-color:#337ab7}.nav .nav-divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5}.nav>li>a>img{max-width:none}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{float:left;margin-bottom:-1px}.nav-tabs>li>a{margin-right:2px;line-height:1.42857143;border:1px solid transparent;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>li.active>a,.nav-tabs>li.active>a:hover,.nav-tabs>li.active>a:focus{color:#555;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent;cursor:default}.nav-tabs.nav-justified{width:100%;border-bottom:0}.nav-tabs.nav-justified>li{float:none}.nav-tabs.nav-justified>li>a
{text-align:center;margin-bottom:5px}.nav-tabs.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-tabs.nav-justified>li{display:table-cell;width:1%}.nav-tabs.nav-justified>li>a{margin-bottom:0}}.nav-tabs.nav-justified>li>a{margin-right:0;border-radius:4px}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:hover,.nav-tabs.nav-justified>.active>a:focus{border:1px solid #ddd}@media (min-width:768px){.nav-tabs.nav-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:hover,.nav-tabs.nav-justified>.active>a:focus{border-bottom-color:#fff}}.nav-pills>li{float:left}.nav-pills>li>a{border-radius:4px}.nav-pills>li+li{margin-left:2px}.nav-pills>li.active>a,.nav-pills>li.active>a:hover,.nav-pills>li.active>a:focus{color:#fff;background-color:#337ab7}.nav-stacked>li{float:none}.nav-stacked>li+li{margin-top:2px;margin-left:0}.nav-justified{width:100%}.nav-just
ified>li{float:none}.nav-justified>li>a{text-align:center;margin-bottom:5px}.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-justified>li{display:table-cell;width:1%}.nav-justified>li>a{margin-bottom:0}}.nav-tabs-justified{border-bottom:0}.nav-tabs-justified>li>a{margin-right:0;border-radius:4px}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:hover,.nav-tabs-justified>.active>a:focus{border:1px solid #ddd}@media (min-width:768px){.nav-tabs-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:hover,.nav-tabs-justified>.active>a:focus{border-bottom-color:#fff}}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-right-radius:0;border-top-left-radius:0}.navbar{position:relative;min-height:50px;margin-bottom:20px;border:1px solid transparent}@media (min-width:768px){.navbar{border-radius:4px
}}@media (min-width:768px){.navbar-header{float:left}}.navbar-collapse{overflow-x:visible;padding-right:15px;padding-left:15px;border-top:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1);-webkit-overflow-scrolling:touch}.navbar-collapse.in{overflow-y:auto}@media (min-width:768px){.navbar-collapse{width:auto;border-top:0;-webkit-box-shadow:none;box-shadow:none}.navbar-collapse.collapse{display:block !important;height:auto !important;padding-bottom:0;overflow:visible !important}.navbar-collapse.in{overflow-y:visible}.navbar-fixed-top .navbar-collapse,.navbar-static-top .navbar-collapse,.navbar-fixed-bottom .navbar-collapse{padding-left:0;padding-right:0}}.navbar-fixed-top .navbar-collapse,.navbar-fixed-bottom .navbar-collapse{max-height:340px}@media (max-device-width:480px) and (orientation:landscape){.navbar-fixed-top .navbar-collapse,.navbar-fixed-bottom .navbar-collapse{max-height:200px}}.container>.navbar-h
eader,.container-fluid>.navbar-header,.container>.navbar-collapse,.container-fluid>.navbar-collapse{margin-right:-15px;margin-left:-15px}@media (min-width:768px){.container>.navbar-header,.container-fluid>.navbar-header,.container>.navbar-collapse,.container-fluid>.navbar-collapse{margin-right:0;margin-left:0}}.navbar-static-top{z-index:1000;border-width:0 0 1px}@media (min-width:768px){.navbar-static-top{border-radius:0}}.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;z-index:1030}@media (min-width:768px){.navbar-fixed-top,.navbar-fixed-bottom{border-radius:0}}.navbar-fixed-top{top:0;border-width:0 0 1px}.navbar-fixed-bottom{bottom:0;margin-bottom:0;border-width:1px 0 0}.navbar-brand{float:left;padding:15px 15px;font-size:18px;line-height:20px;height:50px}.navbar-brand:hover,.navbar-brand:focus{text-decoration:none}.navbar-brand>img{display:block}@media (min-width:768px){.navbar>.container .navbar-brand,.navbar>.container-fluid .navbar-brand{margin-left:-15px}}
.navbar-toggle{position:relative;float:right;margin-right:15px;padding:9px 10px;margin-top:8px;margin-bottom:8px;background-color:transparent;background-image:none;border:1px solid transparent;border-radius:4px}.navbar-toggle:focus{outline:0}.navbar-toggle .icon-bar{display:block;width:22px;height:2px;border-radius:1px}.navbar-toggle .icon-bar+.icon-bar{margin-top:4px}@media (min-width:768px){.navbar-toggle{display:none}}.navbar-nav{margin:7.5px -15px}.navbar-nav>li>a{padding-top:10px;padding-bottom:10px;line-height:20px}@media (max-width:767px){.navbar-nav .open .dropdown-menu{position:static;float:none;width:auto;margin-top:0;background-color:transparent;border:0;-webkit-box-shadow:none;box-shadow:none}.navbar-nav .open .dropdown-menu>li>a,.navbar-nav .open .dropdown-menu .dropdown-header{padding:5px 15px 5px 25px}.navbar-nav .open .dropdown-menu>li>a{line-height:20px}.navbar-nav .open .dropdown-menu>li>a:hover,.navbar-nav .open .dropdown-menu>li>a:focus{background-image:none}}@me
dia (min-width:768px){.navbar-nav{float:left;margin:0}.navbar-nav>li{float:left}.navbar-nav>li>a{padding-top:15px;padding-bottom:15px}}.navbar-form{margin-left:-15px;margin-right:-15px;padding:10px 15px;border-top:1px solid transparent;border-bottom:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);margin-top:8px;margin-bottom:8px}@media (min-width:768px){.navbar-form .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.navbar-form .form-control{display:inline-block;width:auto;vertical-align:middle}.navbar-form .form-control-static{display:inline-block}.navbar-form .input-group{display:inline-table;vertical-align:middle}.navbar-form .input-group .input-group-addon,.navbar-form .input-group .input-group-btn,.navbar-form .input-group .form-control{width:auto}.navbar-form .input-group>.form-control{width:100%}.navbar-form .control-labe
l{margin-bottom:0;vertical-align:middle}.navbar-form .radio,.navbar-form .checkbox{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle}.navbar-form .radio label,.navbar-form .checkbox label{padding-left:0}.navbar-form .radio input[type="radio"],.navbar-form .checkbox input[type="checkbox"]{position:relative;margin-left:0}.navbar-form .has-feedback .form-control-feedback{top:0}}@media (max-width:767px){.navbar-form .form-group{margin-bottom:5px}.navbar-form .form-group:last-child{margin-bottom:0}}@media (min-width:768px){.navbar-form{width:auto;border:0;margin-left:0;margin-right:0;padding-top:0;padding-bottom:0;-webkit-box-shadow:none;box-shadow:none}}.navbar-nav>li>.dropdown-menu{margin-top:0;border-top-right-radius:0;border-top-left-radius:0}.navbar-fixed-bottom .navbar-nav>li>.dropdown-menu{margin-bottom:0;border-top-right-radius:4px;border-top-left-radius:4px;border-bottom-right-radius:0;border-bottom-left-radius:0}.navbar-btn{margin-top:8px;margin-bottom:8px
}.navbar-btn.btn-sm{margin-top:10px;margin-bottom:10px}.navbar-btn.btn-xs{margin-top:14px;margin-bottom:14px}.navbar-text{margin-top:15px;margin-bottom:15px}@media (min-width:768px){.navbar-text{float:left;margin-left:15px;margin-right:15px}}@media (min-width:768px){.navbar-left{float:left !important}.navbar-right{float:right !important;margin-right:-15px}.navbar-right~.navbar-right{margin-right:0}}.navbar-default{background-color:#f8f8f8;border-color:#e7e7e7}.navbar-default .navbar-brand{color:#777}.navbar-default .navbar-brand:hover,.navbar-default .navbar-brand:focus{color:#5e5e5e;background-color:transparent}.navbar-default .navbar-text{color:#777}.navbar-default .navbar-nav>li>a{color:#777}.navbar-default .navbar-nav>li>a:hover,.navbar-default .navbar-nav>li>a:focus{color:#333;background-color:transparent}.navbar-default .navbar-nav>.active>a,.navbar-default .navbar-nav>.active>a:hover,.navbar-default .navbar-nav>.active>a:focus{color:#555;background-color:#e7e7e7}.navbar-defau
lt .navbar-nav>.disabled>a,.navbar-default .navbar-nav>.disabled>a:hover,.navbar-default .navbar-nav>.disabled>a:focus{color:#ccc;background-color:transparent}.navbar-default .navbar-toggle{border-color:#ddd}.navbar-default .navbar-toggle:hover,.navbar-default .navbar-toggle:focus{background-color:#ddd}.navbar-default .navbar-toggle .icon-bar{background-color:#888}.navbar-default .navbar-collapse,.navbar-default .navbar-form{border-color:#e7e7e7}.navbar-default .navbar-nav>.open>a,.navbar-default .navbar-nav>.open>a:hover,.navbar-default .navbar-nav>.open>a:focus{background-color:#e7e7e7;color:#555}@media (max-width:767px){.navbar-default .navbar-nav .open .dropdown-menu>li>a{color:#777}.navbar-default .navbar-nav .open .dropdown-menu>li>a:hover,.navbar-default .navbar-nav .open .dropdown-menu>li>a:focus{color:#333;background-color:transparent}.navbar-default .navbar-nav .open .dropdown-menu>.active>a,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:hover,.navbar-default .
navbar-nav .open .dropdown-menu>.active>a:focus{color:#555;background-color:#e7e7e7}.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:hover,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:focus{color:#ccc;background-color:transparent}}.navbar-default .navbar-link{color:#777}.navbar-default .navbar-link:hover{color:#333}.navbar-default .btn-link{color:#777}.navbar-default .btn-link:hover,.navbar-default .btn-link:focus{color:#333}.navbar-default .btn-link[disabled]:hover,fieldset[disabled] .navbar-default .btn-link:hover,.navbar-default .btn-link[disabled]:focus,fieldset[disabled] .navbar-default .btn-link:focus{color:#ccc}.navbar-inverse{background-color:#222;border-color:#080808}.navbar-inverse .navbar-brand{color:#9d9d9d}.navbar-inverse .navbar-brand:hover,.navbar-inverse .navbar-brand:focus{color:#fff;background-color:transparent}.navbar-inverse .navbar-text{color:#9d9d9d}.navbar-inverse .navbar-
nav>li>a{color:#9d9d9d}.navbar-inverse .navbar-nav>li>a:hover,.navbar-inverse .navbar-nav>li>a:focus{color:#fff;background-color:transparent}.navbar-inverse .navbar-nav>.active>a,.navbar-inverse .navbar-nav>.active>a:hover,.navbar-inverse .navbar-nav>.active>a:focus{color:#fff;background-color:#080808}.navbar-inverse .navbar-nav>.disabled>a,.navbar-inverse .navbar-nav>.disabled>a:hover,.navbar-inverse .navbar-nav>.disabled>a:focus{color:#444;background-color:transparent}.navbar-inverse .navbar-toggle{border-color:#333}.navbar-inverse .navbar-toggle:hover,.navbar-inverse .navbar-toggle:focus{background-color:#333}.navbar-inverse .navbar-toggle .icon-bar{background-color:#fff}.navbar-inverse .navbar-collapse,.navbar-inverse .navbar-form{border-color:#101010}.navbar-inverse .navbar-nav>.open>a,.navbar-inverse .navbar-nav>.open>a:hover,.navbar-inverse .navbar-nav>.open>a:focus{background-color:#080808;color:#fff}@media (max-width:767px){.navbar-inverse .navbar-nav .open .dropdown-menu>.
dropdown-header{border-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu .divider{background-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a{color:#9d9d9d}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:hover,.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:focus{color:#fff;background-color:transparent}.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:hover,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:focus{color:#fff;background-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:hover,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:focus{color:#444;background-color:transparent}}.navbar-inverse .navbar-link{color:#9d9d9d}.navbar-inverse .navbar-link:hover{color:#fff}.navbar-inverse .btn-link{color:#9d9d9d}.navbar-inverse .btn-link:hover,.navbar-inverse .btn-link:f
ocus{color:#fff}.navbar-inverse .btn-link[disabled]:hover,fieldset[disabled] .navbar-inverse .btn-link:hover,.navbar-inverse .btn-link[disabled]:focus,fieldset[disabled] .navbar-inverse .btn-link:focus{color:#444}.breadcrumb{padding:8px 15px;margin-bottom:20px;list-style:none;background-color:#f5f5f5;border-radius:4px}.breadcrumb>li{display:inline-block}.breadcrumb>li+li:before{content:"/\00a0";padding:0 5px;color:#ccc}.breadcrumb>.active{color:#777}.pagination{display:inline-block;padding-left:0;margin:20px 0;border-radius:4px}.pagination>li{display:inline}.pagination>li>a,.pagination>li>span{position:relative;float:left;padding:6px 12px;line-height:1.42857143;text-decoration:none;color:#337ab7;background-color:#fff;border:1px solid #ddd;margin-left:-1px}.pagination>li:first-child>a,.pagination>li:first-child>span{margin-left:0;border-bottom-left-radius:4px;border-top-left-radius:4px}.pagination>li:last-child>a,.pagination>li:last-child>span{border-bottom-right-radius:4px;border-to
p-right-radius:4px}.pagination>li>a:hover,.pagination>li>span:hover,.pagination>li>a:focus,.pagination>li>span:focus{z-index:2;color:#23527c;background-color:#eee;border-color:#ddd}.pagination>.active>a,.pagination>.active>span,.pagination>.active>a:hover,.pagination>.active>span:hover,.pagination>.active>a:focus,.pagination>.active>span:focus{z-index:3;color:#fff;background-color:#337ab7;border-color:#337ab7;cursor:default}.pagination>.disabled>span,.pagination>.disabled>span:hover,.pagination>.disabled>span:focus,.pagination>.disabled>a,.pagination>.disabled>a:hover,.pagination>.disabled>a:focus{color:#777;background-color:#fff;border-color:#ddd;cursor:not-allowed}.pagination-lg>li>a,.pagination-lg>li>span{padding:10px 16px;font-size:18px;line-height:1.3333333}.pagination-lg>li:first-child>a,.pagination-lg>li:first-child>span{border-bottom-left-radius:6px;border-top-left-radius:6px}.pagination-lg>li:last-child>a,.pagination-lg>li:last-child>span{border-bottom-right-radius:6px;bord
er-top-right-radius:6px}.pagination-sm>li>a,.pagination-sm>li>span{padding:5px 10px;font-size:12px;line-height:1.5}.pagination-sm>li:first-child>a,.pagination-sm>li:first-child>span{border-bottom-left-radius:3px;border-top-left-radius:3px}.pagination-sm>li:last-child>a,.pagination-sm>li:last-child>span{border-bottom-right-radius:3px;border-top-right-radius:3px}.pager{padding-left:0;margin:20px 0;list-style:none;text-align:center}.pager li{display:inline}.pager li>a,.pager li>span{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;border-radius:15px}.pager li>a:hover,.pager li>a:focus{text-decoration:none;background-color:#eee}.pager .next>a,.pager .next>span{float:right}.pager .previous>a,.pager .previous>span{float:left}.pager .disabled>a,.pager .disabled>a:hover,.pager .disabled>a:focus,.pager .disabled>span{color:#777;background-color:#fff;cursor:not-allowed}.label{display:inline;padding:.2em .6em .3em;font-size:75%;font-weight:bold;line-height:1;co
lor:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25em}a.label:hover,a.label:focus{color:#fff;text-decoration:none;cursor:pointer}.label:empty{display:none}.btn .label{position:relative;top:-1px}.label-default{background-color:#777}.label-default[href]:hover,.label-default[href]:focus{background-color:#5e5e5e}.label-primary{background-color:#337ab7}.label-primary[href]:hover,.label-primary[href]:focus{background-color:#286090}.label-success{background-color:#5cb85c}.label-success[href]:hover,.label-success[href]:focus{background-color:#449d44}.label-info{background-color:#5bc0de}.label-info[href]:hover,.label-info[href]:focus{background-color:#31b0d5}.label-warning{background-color:#f0ad4e}.label-warning[href]:hover,.label-warning[href]:focus{background-color:#ec971f}.label-danger{background-color:#d9534f}.label-danger[href]:hover,.label-danger[href]:focus{background-color:#c9302c}.badge{display:inline-block;min-width:10px;padding:3px 7px;font-size
:12px;font-weight:bold;color:#fff;line-height:1;vertical-align:middle;white-space:nowrap;text-align:center;background-color:#777;border-radius:10px}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.btn-xs .badge,.btn-group-xs>.btn .badge{top:0;padding:1px 5px}a.badge:hover,a.badge:focus{color:#fff;text-decoration:none;cursor:pointer}.list-group-item.active>.badge,.nav-pills>.active>a>.badge{color:#337ab7;background-color:#fff}.list-group-item>.badge{float:right}.list-group-item>.badge+.badge{margin-right:5px}.nav-pills>li>a>.badge{margin-left:3px}.jumbotron{padding-top:30px;padding-bottom:30px;margin-bottom:30px;color:inherit;background-color:#eee}.jumbotron h1,.jumbotron .h1{color:inherit}.jumbotron p{margin-bottom:15px;font-size:21px;font-weight:200}.jumbotron>hr{border-top-color:#d5d5d5}.container .jumbotron,.container-fluid .jumbotron{border-radius:6px;padding-left:15px;padding-right:15px}.jumbotron .container{max-width:100%}@media screen and (min-width:768px){.j
umbotron{padding-top:48px;padding-bottom:48px}.container .jumbotron,.container-fluid .jumbotron{padding-left:60px;padding-right:60px}.jumbotron h1,.jumbotron .h1{font-size:63px}}.thumbnail{display:block;padding:4px;margin-bottom:20px;line-height:1.42857143;background-color:#fff;border:1px solid #ddd;border-radius:4px;-webkit-transition:border .2s ease-in-out;-o-transition:border .2s ease-in-out;transition:border .2s ease-in-out}.thumbnail>img,.thumbnail a>img{margin-left:auto;margin-right:auto}a.thumbnail:hover,a.thumbnail:focus,a.thumbnail.active{border-color:#337ab7}.thumbnail .caption{padding:9px;color:#333}.alert{padding:15px;margin-bottom:20px;border:1px solid transparent;border-radius:4px}.alert h4{margin-top:0;color:inherit}.alert .alert-link{font-weight:bold}.alert>p,.alert>ul{margin-bottom:0}.alert>p+p{margin-top:5px}.alert-dismissable,.alert-dismissible{padding-right:35px}.alert-dismissable .close,.alert-dismissible .close{position:relative;top:-2px;right:-21px;color:inher
it}.alert-success{background-color:#dff0d8;border-color:#d6e9c6;color:#3c763d}.alert-success hr{border-top-color:#c9e2b3}.alert-success .alert-link{color:#2b542c}.alert-info{background-color:#d9edf7;border-color:#bce8f1;color:#31708f}.alert-info hr{border-top-color:#a6e1ec}.alert-info .alert-link{color:#245269}.alert-warning{background-color:#fcf8e3;border-color:#faebcc;color:#8a6d3b}.alert-warning hr{border-top-color:#f7e1b5}.alert-warning .alert-link{color:#66512c}.alert-danger{background-color:#f2dede;border-color:#ebccd1;color:#a94442}.alert-danger hr{border-top-color:#e4b9c0}.alert-danger .alert-link{color:#843534}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{overflow:hidden;height:20px;margin-bottom:20px;background-color:#f5f5f5;b
order-radius:4px;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1)}.progress-bar{float:left;width:0%;height:100%;font-size:12px;line-height:20px;color:#fff;text-align:center;background-color:#337ab7;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress-striped .progress-bar,.progress-bar-striped{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%,
rgba(255,255,255,0.15) 75%, transparent 75%, transparent);-webkit-background-size:40px 40px;background-size:40px 40px}.progress.active .progress-bar,.progress-bar.active{-webkit-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-bar-success{background-color:#5cb85c}.progress-striped .progress-bar-success{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.pro
gress-bar-info{background-color:#5bc0de}.progress-striped .progress-bar-info{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.progress-bar-warning{background-color:#f0ad4e}.progress-striped .progress-bar-warning{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255,255,255,0.
15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.progress-bar-danger{background-color:#d9534f}.progress-striped .progress-bar-danger{background-image:-webkit-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent)}.media{margin-top:15
px}.media:first-child{margin-top:0}.media,.media-body{zoom:1;overflow:hidden}.media-body{width:10000px}.media-object{display:block}.media-object.img-thumbnail{max-width:none}.media-right,.media>.pull-right{padding-left:10px}.media-left,.media>.pull-left{padding-right:10px}.media-left,.media-right,.media-body{display:table-cell;vertical-align:top}.media-middle{vertical-align:middle}.media-bottom{vertical-align:bottom}.media-heading{margin-top:0;margin-bottom:5px}.media-list{padding-left:0;list-style:none}.list-group{margin-bottom:20px;padding-left:0}.list-group-item{position:relative;display:block;padding:10px 15px;margin-bottom:-1px;background-color:#fff;border:1px solid #ddd}.list-group-item:first-child{border-top-right-radius:4px;border-top-left-radius:4px}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:4px;border-bottom-left-radius:4px}a.list-group-item,button.list-group-item{color:#555}a.list-group-item .list-group-item-heading,button.list-group-item .list
-group-item-heading{color:#333}a.list-group-item:hover,button.list-group-item:hover,a.list-group-item:focus,button.list-group-item:focus{text-decoration:none;color:#555;background-color:#f5f5f5}button.list-group-item{width:100%;text-align:left}.list-group-item.disabled,.list-group-item.disabled:hover,.list-group-item.disabled:focus{background-color:#eee;color:#777;cursor:not-allowed}.list-group-item.disabled .list-group-item-heading,.list-group-item.disabled:hover .list-group-item-heading,.list-group-item.disabled:focus .list-group-item-heading{color:inherit}.list-group-item.disabled .list-group-item-text,.list-group-item.disabled:hover .list-group-item-text,.list-group-item.disabled:focus .list-group-item-text{color:#777}.list-group-item.active,.list-group-item.active:hover,.list-group-item.active:focus{z-index:2;color:#fff;background-color:#337ab7;border-color:#337ab7}.list-group-item.active .list-group-item-heading,.list-group-item.active:hover .list-group-item-heading,.list-grou
p-item.active:focus .list-group-item-heading,.list-group-item.active .list-group-item-heading>small,.list-group-item.active:hover .list-group-item-heading>small,.list-group-item.active:focus .list-group-item-heading>small,.list-group-item.active .list-group-item-heading>.small,.list-group-item.active:hover .list-group-item-heading>.small,.list-group-item.active:focus .list-group-item-heading>.small{color:inherit}.list-group-item.active .list-group-item-text,.list-group-item.active:hover .list-group-item-text,.list-group-item.active:focus .list-group-item-text{color:#c7ddef}.list-group-item-success{color:#3c763d;background-color:#dff0d8}a.list-group-item-success,button.list-group-item-success{color:#3c763d}a.list-group-item-success .list-group-item-heading,button.list-group-item-success .list-group-item-heading{color:inherit}a.list-group-item-success:hover,button.list-group-item-success:hover,a.list-group-item-success:focus,button.list-group-item-success:focus{color:#3c763d;backgroun
d-color:#d0e9c6}a.list-group-item-success.active,button.list-group-item-success.active,a.list-group-item-success.active:hover,button.list-group-item-success.active:hover,a.list-group-item-success.active:focus,button.list-group-item-success.active:focus{color:#fff;background-color:#3c763d;border-color:#3c763d}.list-group-item-info{color:#31708f;background-color:#d9edf7}a.list-group-item-info,button.list-group-item-info{color:#31708f}a.list-group-item-info .list-group-item-heading,button.list-group-item-info .list-group-item-heading{color:inherit}a.list-group-item-info:hover,button.list-group-item-info:hover,a.list-group-item-info:focus,button.list-group-item-info:focus{color:#31708f;background-color:#c4e3f3}a.list-group-item-info.active,button.list-group-item-info.active,a.list-group-item-info.active:hover,button.list-group-item-info.active:hover,a.list-group-item-info.active:focus,button.list-group-item-info.active:focus{color:#fff;background-color:#31708f;border-color:#31708f}.list
-group-item-warning{color:#8a6d3b;background-color:#fcf8e3}a.list-group-item-warning,button.list-group-item-warning{color:#8a6d3b}a.list-group-item-warning .list-group-item-heading,button.list-group-item-warning .list-group-item-heading{color:inherit}a.list-group-item-warning:hover,button.list-group-item-warning:hover,a.list-group-item-warning:focus,button.list-group-item-warning:focus{color:#8a6d3b;background-color:#faf2cc}a.list-group-item-warning.active,button.list-group-item-warning.active,a.list-group-item-warning.active:hover,button.list-group-item-warning.active:hover,a.list-group-item-warning.active:focus,button.list-group-item-warning.active:focus{color:#fff;background-color:#8a6d3b;border-color:#8a6d3b}.list-group-item-danger{color:#a94442;background-color:#f2dede}a.list-group-item-danger,button.list-group-item-danger{color:#a94442}a.list-group-item-danger .list-group-item-heading,button.list-group-item-danger .list-group-item-heading{color:inherit}a.list-group-item-danger
:hover,button.list-group-item-danger:hover,a.list-group-item-danger:focus,button.list-group-item-danger:focus{color:#a94442;background-color:#ebcccc}a.list-group-item-danger.active,button.list-group-item-danger.active,a.list-group-item-danger.active:hover,button.list-group-item-danger.active:hover,a.list-group-item-danger.active:focus,button.list-group-item-danger.active:focus{color:#fff;background-color:#a94442;border-color:#a94442}.list-group-item-heading{margin-top:0;margin-bottom:5px}.list-group-item-text{margin-bottom:0;line-height:1.3}.panel{margin-bottom:20px;background-color:#fff;border:1px solid transparent;border-radius:4px;-webkit-box-shadow:0 1px 1px rgba(0,0,0,0.05);box-shadow:0 1px 1px rgba(0,0,0,0.05)}.panel-body{padding:15px}.panel-heading{padding:10px 15px;border-bottom:1px solid transparent;border-top-right-radius:3px;border-top-left-radius:3px}.panel-heading>.dropdown .dropdown-toggle{color:inherit}.panel-title{margin-top:0;margin-bottom:0;font-size:16px;color:inh
erit}.panel-title>a,.panel-title>small,.panel-title>.small,.panel-title>small>a,.panel-title>.small>a{color:inherit}.panel-footer{padding:10px 15px;background-color:#f5f5f5;border-top:1px solid #ddd;border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.list-group,.panel>.panel-collapse>.list-group{margin-bottom:0}.panel>.list-group .list-group-item,.panel>.panel-collapse>.list-group .list-group-item{border-width:1px 0;border-radius:0}.panel>.list-group:first-child .list-group-item:first-child,.panel>.panel-collapse>.list-group:first-child .list-group-item:first-child{border-top:0;border-top-right-radius:3px;border-top-left-radius:3px}.panel>.list-group:last-child .list-group-item:last-child,.panel>.panel-collapse>.list-group:last-child .list-group-item:last-child{border-bottom:0;border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.panel-heading+.panel-collapse>.list-group .list-group-item:first-child{border-top-right-radius:0;border-top-left-radius:0}.
panel-heading+.list-group .list-group-item:first-child{border-top-width:0}.list-group+.panel-footer{border-top-width:0}.panel>.table,.panel>.table-responsive>.table,.panel>.panel-collapse>.table{margin-bottom:0}.panel>.table caption,.panel>.table-responsive>.table caption,.panel>.panel-collapse>.table caption{padding-left:15px;padding-right:15px}.panel>.table:first-child,.panel>.table-responsive:first-child>.table:first-child{border-top-right-radius:3px;border-top-left-radius:3px}.panel>.table:first-child>thead:first-child>tr:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child{border-top-left-radius:3px;border-top-right-radius:3px}.panel>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-chil
d td:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table:first-child>thead:first-child>tr:first-child th:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:first-child{border-top-left-radius:3px}.panel>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table:first-child>thead:first-child>tr:first-child th:last-
child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:last-child{border-top-right-radius:3px}.panel>.table:last-child,.panel>.table-responsive:last-child>.table:last-child{border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.table:last-child>tbody:last-child>tr:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child{border-bottom-left-radius:3px;border-bottom-right-radius:3px}.panel>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table:last-child>tfoot
:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:first-child{border-bottom-left-radius:3px}.panel>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:la
st-child th:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:last-child{border-bottom-right-radius:3px}.panel>.panel-body+.table,.panel>.panel-body+.table-responsive,.panel>.table+.panel-body,.panel>.table-responsive+.panel-body{border-top:1px solid #ddd}.panel>.table>tbody:first-child>tr:first-child th,.panel>.table>tbody:first-child>tr:first-child td{border-top:0}.panel>.table-bordered,.panel>.table-responsive>.table-bordered{border:0}.panel>.table-bordered>thead>tr>th:first-child,.panel>.table-responsive>.table-bord
<TRUNCATED>
[10/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/jquery.dataTables.min.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/jquery.dataTables.min.js b/server/src/main/resources/org/apache/livy/server/ui/static/jquery.dataTables.min.js
new file mode 100644
index 0000000..dc969ee
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/jquery.dataTables.min.js
@@ -0,0 +1,167 @@
+/*!
+ DataTables 1.10.15
+ ©2008-2017 SpryMedia Ltd - datatables.net/license
+*/
+(function(h){"function"===typeof define&&define.amd?define(["jquery"],function(E){return h(E,window,document)}):"object"===typeof exports?module.exports=function(E,H){E||(E=window);H||(H="undefined"!==typeof window?require("jquery"):require("jquery")(E));return h(H,E,E.document)}:h(jQuery,window,document)})(function(h,E,H,k){function Y(a){var b,c,d={};h.each(a,function(e){if((b=e.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" "))c=e.replace(b[0],b[2].toLowerCase()),
+d[c]=e,"o"===b[1]&&Y(a[e])});a._hungarianMap=d}function J(a,b,c){a._hungarianMap||Y(a);var d;h.each(b,function(e){d=a._hungarianMap[e];if(d!==k&&(c||b[d]===k))"o"===d.charAt(0)?(b[d]||(b[d]={}),h.extend(!0,b[d],b[e]),J(a[d],b[d],c)):b[d]=b[e]})}function Fa(a){var b=m.defaults.oLanguage,c=a.sZeroRecords;!a.sEmptyTable&&(c&&"No data available in table"===b.sEmptyTable)&&F(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&(c&&"Loading..."===b.sLoadingRecords)&&F(a,a,"sZeroRecords","sLoadingRecords");
+a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&fb(a)}function gb(a){A(a,"ordering","bSort");A(a,"orderMulti","bSortMulti");A(a,"orderClasses","bSortClasses");A(a,"orderCellsTop","bSortCellsTop");A(a,"order","aaSorting");A(a,"orderFixed","aaSortingFixed");A(a,"paging","bPaginate");A(a,"pagingType","sPaginationType");A(a,"pageLength","iDisplayLength");A(a,"searching","bFilter");"boolean"===typeof a.sScrollX&&(a.sScrollX=a.sScrollX?"100%":"");"boolean"===typeof a.scrollX&&(a.scrollX=
+a.scrollX?"100%":"");if(a=a.aoSearchCols)for(var b=0,c=a.length;b<c;b++)a[b]&&J(m.models.oSearch,a[b])}function hb(a){A(a,"orderable","bSortable");A(a,"orderData","aDataSort");A(a,"orderSequence","asSorting");A(a,"orderDataType","sortDataType");var b=a.aDataSort;"number"===typeof b&&!h.isArray(b)&&(a.aDataSort=[b])}function ib(a){if(!m.__browser){var b={};m.__browser=b;var c=h("<div/>").css({position:"fixed",top:0,left:-1*h(E).scrollLeft(),height:1,width:1,overflow:"hidden"}).append(h("<div/>").css({position:"absolute",
+top:1,left:1,width:100,overflow:"scroll"}).append(h("<div/>").css({width:"100%",height:10}))).appendTo("body"),d=c.children(),e=d.children();b.barWidth=d[0].offsetWidth-d[0].clientWidth;b.bScrollOversize=100===e[0].offsetWidth&&100!==d[0].clientWidth;b.bScrollbarLeft=1!==Math.round(e.offset().left);b.bBounding=c[0].getBoundingClientRect().width?!0:!1;c.remove()}h.extend(a.oBrowser,m.__browser);a.oScroll.iBarWidth=m.__browser.barWidth}function jb(a,b,c,d,e,f){var g,j=!1;c!==k&&(g=c,j=!0);for(;d!==
+e;)a.hasOwnProperty(d)&&(g=j?b(g,a[d],d,a):a[d],j=!0,d+=f);return g}function Ga(a,b){var c=m.defaults.column,d=a.aoColumns.length,c=h.extend({},m.models.oColumn,c,{nTh:b?b:H.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.mData:d,idx:d});a.aoColumns.push(c);c=a.aoPreSearchCols;c[d]=h.extend({},m.models.oSearch,c[d]);la(a,d,h(b).data())}function la(a,b,c){var b=a.aoColumns[b],d=a.oClasses,e=h(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=
+e.attr("width")||null;var f=(e.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);f&&(b.sWidthOrig=f[1])}c!==k&&null!==c&&(hb(c),J(m.defaults.column,c),c.mDataProp!==k&&!c.mData&&(c.mData=c.mDataProp),c.sType&&(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),h.extend(b,c),F(b,c,"sWidth","sWidthOrig"),c.iDataSort!==k&&(b.aDataSort=[c.iDataSort]),F(b,c,"aDataSort"));var g=b.mData,j=R(g),i=b.mRender?R(b.mRender):null,c=function(a){return"string"===typeof a&&-1!==a.indexOf("@")};
+b._bAttrSrc=h.isPlainObject(g)&&(c(g.sort)||c(g.type)||c(g.filter));b._setter=null;b.fnGetData=function(a,b,c){var d=j(a,b,k,c);return i&&b?i(d,b,a,c):d};b.fnSetData=function(a,b,c){return S(g)(a,b,c)};"number"!==typeof g&&(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,e.addClass(d.sSortableNone));a=-1!==h.inArray("asc",b.asSorting);c=-1!==h.inArray("desc",b.asSorting);!b.bSortable||!a&&!c?(b.sSortingClass=d.sSortableNone,b.sSortingClassJUI=""):a&&!c?(b.sSortingClass=d.sSortableAsc,b.sSortingClassJUI=
+d.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=d.sSortableDesc,b.sSortingClassJUI=d.sSortJUIDescAllowed):(b.sSortingClass=d.sSortable,b.sSortingClassJUI=d.sSortJUI)}function Z(a){if(!1!==a.oFeatures.bAutoWidth){var b=a.aoColumns;Ha(a);for(var c=0,d=b.length;c<d;c++)b[c].nTh.style.width=b[c].sWidth}b=a.oScroll;(""!==b.sY||""!==b.sX)&&ma(a);s(a,null,"column-sizing",[a])}function $(a,b){var c=na(a,"bVisible");return"number"===typeof c[b]?c[b]:null}function aa(a,b){var c=na(a,"bVisible"),c=h.inArray(b,
+c);return-1!==c?c:null}function ba(a){var b=0;h.each(a.aoColumns,function(a,d){d.bVisible&&"none"!==h(d.nTh).css("display")&&b++});return b}function na(a,b){var c=[];h.map(a.aoColumns,function(a,e){a[b]&&c.push(e)});return c}function Ia(a){var b=a.aoColumns,c=a.aoData,d=m.ext.type.detect,e,f,g,j,i,h,l,q,r;e=0;for(f=b.length;e<f;e++)if(l=b[e],r=[],!l.sType&&l._sManualType)l.sType=l._sManualType;else if(!l.sType){g=0;for(j=d.length;g<j;g++){i=0;for(h=c.length;i<h;i++){r[i]===k&&(r[i]=B(a,i,e,"type"));
+q=d[g](r[i],a);if(!q&&g!==d.length-1)break;if("html"===q)break}if(q){l.sType=q;break}}l.sType||(l.sType="string")}}function kb(a,b,c,d){var e,f,g,j,i,n,l=a.aoColumns;if(b)for(e=b.length-1;0<=e;e--){n=b[e];var q=n.targets!==k?n.targets:n.aTargets;h.isArray(q)||(q=[q]);f=0;for(g=q.length;f<g;f++)if("number"===typeof q[f]&&0<=q[f]){for(;l.length<=q[f];)Ga(a);d(q[f],n)}else if("number"===typeof q[f]&&0>q[f])d(l.length+q[f],n);else if("string"===typeof q[f]){j=0;for(i=l.length;j<i;j++)("_all"==q[f]||h(l[j].nTh).hasClass(q[f]))&&
+d(j,n)}}if(c){e=0;for(a=c.length;e<a;e++)d(e,c[e])}}function N(a,b,c,d){var e=a.aoData.length,f=h.extend(!0,{},m.models.oRow,{src:c?"dom":"data",idx:e});f._aData=b;a.aoData.push(f);for(var g=a.aoColumns,j=0,i=g.length;j<i;j++)g[j].sType=null;a.aiDisplayMaster.push(e);b=a.rowIdFn(b);b!==k&&(a.aIds[b]=f);(c||!a.oFeatures.bDeferRender)&&Ja(a,e,c,d);return e}function oa(a,b){var c;b instanceof h||(b=h(b));return b.map(function(b,e){c=Ka(a,e);return N(a,c.data,e,c.cells)})}function B(a,b,c,d){var e=a.iDraw,
+f=a.aoColumns[c],g=a.aoData[b]._aData,j=f.sDefaultContent,i=f.fnGetData(g,d,{settings:a,row:b,col:c});if(i===k)return a.iDrawError!=e&&null===j&&(K(a,0,"Requested unknown parameter "+("function"==typeof f.mData?"{function}":"'"+f.mData+"'")+" for row "+b+", column "+c,4),a.iDrawError=e),j;if((i===g||null===i)&&null!==j&&d!==k)i=j;else if("function"===typeof i)return i.call(g);return null===i&&"display"==d?"":i}function lb(a,b,c,d){a.aoColumns[c].fnSetData(a.aoData[b]._aData,d,{settings:a,row:b,col:c})}
+function La(a){return h.map(a.match(/(\\.|[^\.])+/g)||[""],function(a){return a.replace(/\\\./g,".")})}function R(a){if(h.isPlainObject(a)){var b={};h.each(a,function(a,c){c&&(b[a]=R(c))});return function(a,c,f,g){var j=b[c]||b._;return j!==k?j(a,c,f,g):a}}if(null===a)return function(a){return a};if("function"===typeof a)return function(b,c,f,g){return a(b,c,f,g)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var c=function(a,b,f){var g,j;if(""!==f){j=La(f);
+for(var i=0,n=j.length;i<n;i++){f=j[i].match(ca);g=j[i].match(V);if(f){j[i]=j[i].replace(ca,"");""!==j[i]&&(a=a[j[i]]);g=[];j.splice(0,i+1);j=j.join(".");if(h.isArray(a)){i=0;for(n=a.length;i<n;i++)g.push(c(a[i],b,j))}a=f[0].substring(1,f[0].length-1);a=""===a?g:g.join(a);break}else if(g){j[i]=j[i].replace(V,"");a=a[j[i]]();continue}if(null===a||a[j[i]]===k)return k;a=a[j[i]]}}return a};return function(b,e){return c(b,e,a)}}return function(b){return b[a]}}function S(a){if(h.isPlainObject(a))return S(a._);
+if(null===a)return function(){};if("function"===typeof a)return function(b,d,e){a(b,"set",d,e)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var b=function(a,d,e){var e=La(e),f;f=e[e.length-1];for(var g,j,i=0,n=e.length-1;i<n;i++){g=e[i].match(ca);j=e[i].match(V);if(g){e[i]=e[i].replace(ca,"");a[e[i]]=[];f=e.slice();f.splice(0,i+1);g=f.join(".");if(h.isArray(d)){j=0;for(n=d.length;j<n;j++)f={},b(f,d[j],g),a[e[i]].push(f)}else a[e[i]]=d;return}j&&(e[i]=e[i].replace(V,
+""),a=a[e[i]](d));if(null===a[e[i]]||a[e[i]]===k)a[e[i]]={};a=a[e[i]]}if(f.match(V))a[f.replace(V,"")](d);else a[f.replace(ca,"")]=d};return function(c,d){return b(c,d,a)}}return function(b,d){b[a]=d}}function Ma(a){return D(a.aoData,"_aData")}function pa(a){a.aoData.length=0;a.aiDisplayMaster.length=0;a.aiDisplay.length=0;a.aIds={}}function qa(a,b,c){for(var d=-1,e=0,f=a.length;e<f;e++)a[e]==b?d=e:a[e]>b&&a[e]--; -1!=d&&c===k&&a.splice(d,1)}function da(a,b,c,d){var e=a.aoData[b],f,g=function(c,d){for(;c.childNodes.length;)c.removeChild(c.firstChild);
+c.innerHTML=B(a,b,d,"display")};if("dom"===c||(!c||"auto"===c)&&"dom"===e.src)e._aData=Ka(a,e,d,d===k?k:e._aData).data;else{var j=e.anCells;if(j)if(d!==k)g(j[d],d);else{c=0;for(f=j.length;c<f;c++)g(j[c],c)}}e._aSortData=null;e._aFilterData=null;g=a.aoColumns;if(d!==k)g[d].sType=null;else{c=0;for(f=g.length;c<f;c++)g[c].sType=null;Na(a,e)}}function Ka(a,b,c,d){var e=[],f=b.firstChild,g,j,i=0,n,l=a.aoColumns,q=a._rowReadObject,d=d!==k?d:q?{}:[],r=function(a,b){if("string"===typeof a){var c=a.indexOf("@");
+-1!==c&&(c=a.substring(c+1),S(a)(d,b.getAttribute(c)))}},m=function(a){if(c===k||c===i)j=l[i],n=h.trim(a.innerHTML),j&&j._bAttrSrc?(S(j.mData._)(d,n),r(j.mData.sort,a),r(j.mData.type,a),r(j.mData.filter,a)):q?(j._setter||(j._setter=S(j.mData)),j._setter(d,n)):d[i]=n;i++};if(f)for(;f;){g=f.nodeName.toUpperCase();if("TD"==g||"TH"==g)m(f),e.push(f);f=f.nextSibling}else{e=b.anCells;f=0;for(g=e.length;f<g;f++)m(e[f])}if(b=b.firstChild?b:b.nTr)(b=b.getAttribute("id"))&&S(a.rowId)(d,b);return{data:d,cells:e}}
+function Ja(a,b,c,d){var e=a.aoData[b],f=e._aData,g=[],j,i,n,l,q;if(null===e.nTr){j=c||H.createElement("tr");e.nTr=j;e.anCells=g;j._DT_RowIndex=b;Na(a,e);l=0;for(q=a.aoColumns.length;l<q;l++){n=a.aoColumns[l];i=c?d[l]:H.createElement(n.sCellType);i._DT_CellIndex={row:b,column:l};g.push(i);if((!c||n.mRender||n.mData!==l)&&(!h.isPlainObject(n.mData)||n.mData._!==l+".display"))i.innerHTML=B(a,b,l,"display");n.sClass&&(i.className+=" "+n.sClass);n.bVisible&&!c?j.appendChild(i):!n.bVisible&&c&&i.parentNode.removeChild(i);
+n.fnCreatedCell&&n.fnCreatedCell.call(a.oInstance,i,B(a,b,l),f,b,l)}s(a,"aoRowCreatedCallback",null,[j,f,b])}e.nTr.setAttribute("role","row")}function Na(a,b){var c=b.nTr,d=b._aData;if(c){var e=a.rowIdFn(d);e&&(c.id=e);d.DT_RowClass&&(e=d.DT_RowClass.split(" "),b.__rowc=b.__rowc?sa(b.__rowc.concat(e)):e,h(c).removeClass(b.__rowc.join(" ")).addClass(d.DT_RowClass));d.DT_RowAttr&&h(c).attr(d.DT_RowAttr);d.DT_RowData&&h(c).data(d.DT_RowData)}}function mb(a){var b,c,d,e,f,g=a.nTHead,j=a.nTFoot,i=0===
+h("th, td",g).length,n=a.oClasses,l=a.aoColumns;i&&(e=h("<tr/>").appendTo(g));b=0;for(c=l.length;b<c;b++)f=l[b],d=h(f.nTh).addClass(f.sClass),i&&d.appendTo(e),a.oFeatures.bSort&&(d.addClass(f.sSortingClass),!1!==f.bSortable&&(d.attr("tabindex",a.iTabIndex).attr("aria-controls",a.sTableId),Oa(a,f.nTh,b))),f.sTitle!=d[0].innerHTML&&d.html(f.sTitle),Pa(a,"header")(a,d,f,n);i&&ea(a.aoHeader,g);h(g).find(">tr").attr("role","row");h(g).find(">tr>th, >tr>td").addClass(n.sHeaderTH);h(j).find(">tr>th, >tr>td").addClass(n.sFooterTH);
+if(null!==j){a=a.aoFooter[0];b=0;for(c=a.length;b<c;b++)f=l[b],f.nTf=a[b].cell,f.sClass&&h(f.nTf).addClass(f.sClass)}}function fa(a,b,c){var d,e,f,g=[],j=[],i=a.aoColumns.length,n;if(b){c===k&&(c=!1);d=0;for(e=b.length;d<e;d++){g[d]=b[d].slice();g[d].nTr=b[d].nTr;for(f=i-1;0<=f;f--)!a.aoColumns[f].bVisible&&!c&&g[d].splice(f,1);j.push([])}d=0;for(e=g.length;d<e;d++){if(a=g[d].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[d].length;f<b;f++)if(n=i=1,j[d][f]===k){a.appendChild(g[d][f].cell);
+for(j[d][f]=1;g[d+i]!==k&&g[d][f].cell==g[d+i][f].cell;)j[d+i][f]=1,i++;for(;g[d][f+n]!==k&&g[d][f].cell==g[d][f+n].cell;){for(c=0;c<i;c++)j[d+c][f+n]=1;n++}h(g[d][f].cell).attr("rowspan",i).attr("colspan",n)}}}}function O(a){var b=s(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==h.inArray(!1,b))C(a,!1);else{var b=[],c=0,d=a.asStripeClasses,e=d.length,f=a.oLanguage,g=a.iInitDisplayStart,j="ssp"==y(a),i=a.aiDisplay;a.bDrawing=!0;g!==k&&-1!==g&&(a._iDisplayStart=j?g:g>=a.fnRecordsDisplay()?0:g,a.iInitDisplayStart=
+-1);var g=a._iDisplayStart,n=a.fnDisplayEnd();if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,C(a,!1);else if(j){if(!a.bDestroying&&!nb(a))return}else a.iDraw++;if(0!==i.length){f=j?a.aoData.length:n;for(j=j?0:g;j<f;j++){var l=i[j],q=a.aoData[l];null===q.nTr&&Ja(a,l);l=q.nTr;if(0!==e){var r=d[c%e];q._sRowStripe!=r&&(h(l).removeClass(q._sRowStripe).addClass(r),q._sRowStripe=r)}s(a,"aoRowCallback",null,[l,q._aData,c,j]);b.push(l);c++}}else c=f.sZeroRecords,1==a.iDraw&&"ajax"==y(a)?c=f.sLoadingRecords:
+f.sEmptyTable&&0===a.fnRecordsTotal()&&(c=f.sEmptyTable),b[0]=h("<tr/>",{"class":e?d[0]:""}).append(h("<td />",{valign:"top",colSpan:ba(a),"class":a.oClasses.sRowEmpty}).html(c))[0];s(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],Ma(a),g,n,i]);s(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],Ma(a),g,n,i]);d=h(a.nTBody);d.children().detach();d.append(h(b));s(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1}}function T(a,b){var c=a.oFeatures,d=c.bFilter;
+c.bSort&&ob(a);d?ga(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;O(a);a._drawHold=!1}function pb(a){var b=a.oClasses,c=h(a.nTable),c=h("<div/>").insertBefore(c),d=a.oFeatures,e=h("<div/>",{id:a.sTableId+"_wrapper","class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=e[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var f=a.sDom.split(""),g,j,i,n,l,q,k=0;k<f.length;k++){g=null;j=f[k];if("<"==j){i=h("<div/>")[0];
+n=f[k+1];if("'"==n||'"'==n){l="";for(q=2;f[k+q]!=n;)l+=f[k+q],q++;"H"==l?l=b.sJUIHeader:"F"==l&&(l=b.sJUIFooter);-1!=l.indexOf(".")?(n=l.split("."),i.id=n[0].substr(1,n[0].length-1),i.className=n[1]):"#"==l.charAt(0)?i.id=l.substr(1,l.length-1):i.className=l;k+=q}e.append(i);e=h(i)}else if(">"==j)e=e.parent();else if("l"==j&&d.bPaginate&&d.bLengthChange)g=qb(a);else if("f"==j&&d.bFilter)g=rb(a);else if("r"==j&&d.bProcessing)g=sb(a);else if("t"==j)g=tb(a);else if("i"==j&&d.bInfo)g=ub(a);else if("p"==
+j&&d.bPaginate)g=vb(a);else if(0!==m.ext.feature.length){i=m.ext.feature;q=0;for(n=i.length;q<n;q++)if(j==i[q].cFeature){g=i[q].fnInit(a);break}}g&&(i=a.aanFeatures,i[j]||(i[j]=[]),i[j].push(g),e.append(g))}c.replaceWith(e);a.nHolding=null}function ea(a,b){var c=h(b).children("tr"),d,e,f,g,j,i,n,l,q,k;a.splice(0,a.length);f=0;for(i=c.length;f<i;f++)a.push([]);f=0;for(i=c.length;f<i;f++){d=c[f];for(e=d.firstChild;e;){if("TD"==e.nodeName.toUpperCase()||"TH"==e.nodeName.toUpperCase()){l=1*e.getAttribute("colspan");
+q=1*e.getAttribute("rowspan");l=!l||0===l||1===l?1:l;q=!q||0===q||1===q?1:q;g=0;for(j=a[f];j[g];)g++;n=g;k=1===l?!0:!1;for(j=0;j<l;j++)for(g=0;g<q;g++)a[f+g][n+j]={cell:e,unique:k},a[f+g].nTr=d}e=e.nextSibling}}}function ta(a,b,c){var d=[];c||(c=a.aoHeader,b&&(c=[],ea(c,b)));for(var b=0,e=c.length;b<e;b++)for(var f=0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!d[f]||!a.bSortCellsTop))d[f]=c[b][f].cell;return d}function ua(a,b,c){s(a,"aoServerParams","serverParams",[b]);if(b&&h.isArray(b)){var d={},
+e=/(.*?)\[\]$/;h.each(b,function(a,b){var c=b.name.match(e);c?(c=c[0],d[c]||(d[c]=[]),d[c].push(b.value)):d[b.name]=b.value});b=d}var f,g=a.ajax,j=a.oInstance,i=function(b){s(a,null,"xhr",[a,b,a.jqXHR]);c(b)};if(h.isPlainObject(g)&&g.data){f=g.data;var n=h.isFunction(f)?f(b,a):f,b=h.isFunction(f)&&n?n:h.extend(!0,b,n);delete g.data}n={data:b,success:function(b){var c=b.error||b.sError;c&&K(a,0,c);a.json=b;i(b)},dataType:"json",cache:!1,type:a.sServerMethod,error:function(b,c){var d=s(a,null,"xhr",
+[a,null,a.jqXHR]);-1===h.inArray(!0,d)&&("parsererror"==c?K(a,0,"Invalid JSON response",1):4===b.readyState&&K(a,0,"Ajax error",7));C(a,!1)}};a.oAjaxData=b;s(a,null,"preXhr",[a,b]);a.fnServerData?a.fnServerData.call(j,a.sAjaxSource,h.map(b,function(a,b){return{name:b,value:a}}),i,a):a.sAjaxSource||"string"===typeof g?a.jqXHR=h.ajax(h.extend(n,{url:g||a.sAjaxSource})):h.isFunction(g)?a.jqXHR=g.call(j,b,i,a):(a.jqXHR=h.ajax(h.extend(n,g)),g.data=f)}function nb(a){return a.bAjaxDataGet?(a.iDraw++,C(a,
+!0),ua(a,wb(a),function(b){xb(a,b)}),!1):!0}function wb(a){var b=a.aoColumns,c=b.length,d=a.oFeatures,e=a.oPreviousSearch,f=a.aoPreSearchCols,g,j=[],i,n,l,k=W(a);g=a._iDisplayStart;i=!1!==d.bPaginate?a._iDisplayLength:-1;var r=function(a,b){j.push({name:a,value:b})};r("sEcho",a.iDraw);r("iColumns",c);r("sColumns",D(b,"sName").join(","));r("iDisplayStart",g);r("iDisplayLength",i);var ra={draw:a.iDraw,columns:[],order:[],start:g,length:i,search:{value:e.sSearch,regex:e.bRegex}};for(g=0;g<c;g++)n=b[g],
+l=f[g],i="function"==typeof n.mData?"function":n.mData,ra.columns.push({data:i,name:n.sName,searchable:n.bSearchable,orderable:n.bSortable,search:{value:l.sSearch,regex:l.bRegex}}),r("mDataProp_"+g,i),d.bFilter&&(r("sSearch_"+g,l.sSearch),r("bRegex_"+g,l.bRegex),r("bSearchable_"+g,n.bSearchable)),d.bSort&&r("bSortable_"+g,n.bSortable);d.bFilter&&(r("sSearch",e.sSearch),r("bRegex",e.bRegex));d.bSort&&(h.each(k,function(a,b){ra.order.push({column:b.col,dir:b.dir});r("iSortCol_"+a,b.col);r("sSortDir_"+
+a,b.dir)}),r("iSortingCols",k.length));b=m.ext.legacy.ajax;return null===b?a.sAjaxSource?j:ra:b?j:ra}function xb(a,b){var c=va(a,b),d=b.sEcho!==k?b.sEcho:b.draw,e=b.iTotalRecords!==k?b.iTotalRecords:b.recordsTotal,f=b.iTotalDisplayRecords!==k?b.iTotalDisplayRecords:b.recordsFiltered;if(d){if(1*d<a.iDraw)return;a.iDraw=1*d}pa(a);a._iRecordsTotal=parseInt(e,10);a._iRecordsDisplay=parseInt(f,10);d=0;for(e=c.length;d<e;d++)N(a,c[d]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=!1;O(a);a._bInitComplete||
+wa(a,b);a.bAjaxDataGet=!0;C(a,!1)}function va(a,b){var c=h.isPlainObject(a.ajax)&&a.ajax.dataSrc!==k?a.ajax.dataSrc:a.sAjaxDataProp;return"data"===c?b.aaData||b[c]:""!==c?R(c)(b):b}function rb(a){var b=a.oClasses,c=a.sTableId,d=a.oLanguage,e=a.oPreviousSearch,f=a.aanFeatures,g='<input type="search" class="'+b.sFilterInput+'"/>',j=d.sSearch,j=j.match(/_INPUT_/)?j.replace("_INPUT_",g):j+g,b=h("<div/>",{id:!f.f?c+"_filter":null,"class":b.sFilter}).append(h("<label/>").append(j)),f=function(){var b=!this.value?
+"":this.value;b!=e.sSearch&&(ga(a,{sSearch:b,bRegex:e.bRegex,bSmart:e.bSmart,bCaseInsensitive:e.bCaseInsensitive}),a._iDisplayStart=0,O(a))},g=null!==a.searchDelay?a.searchDelay:"ssp"===y(a)?400:0,i=h("input",b).val(e.sSearch).attr("placeholder",d.sSearchPlaceholder).on("keyup.DT search.DT input.DT paste.DT cut.DT",g?Qa(f,g):f).on("keypress.DT",function(a){if(13==a.keyCode)return!1}).attr("aria-controls",c);h(a.nTable).on("search.dt.DT",function(b,c){if(a===c)try{i[0]!==H.activeElement&&i.val(e.sSearch)}catch(d){}});
+return b[0]}function ga(a,b,c){var d=a.oPreviousSearch,e=a.aoPreSearchCols,f=function(a){d.sSearch=a.sSearch;d.bRegex=a.bRegex;d.bSmart=a.bSmart;d.bCaseInsensitive=a.bCaseInsensitive};Ia(a);if("ssp"!=y(a)){yb(a,b.sSearch,c,b.bEscapeRegex!==k?!b.bEscapeRegex:b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<e.length;b++)zb(a,e[b].sSearch,b,e[b].bEscapeRegex!==k?!e[b].bEscapeRegex:e[b].bRegex,e[b].bSmart,e[b].bCaseInsensitive);Ab(a)}else f(b);a.bFiltered=!0;s(a,null,"search",[a])}function Ab(a){for(var b=
+m.ext.search,c=a.aiDisplay,d,e,f=0,g=b.length;f<g;f++){for(var j=[],i=0,n=c.length;i<n;i++)e=c[i],d=a.aoData[e],b[f](a,d._aFilterData,e,d._aData,i)&&j.push(e);c.length=0;h.merge(c,j)}}function zb(a,b,c,d,e,f){if(""!==b){for(var g=[],j=a.aiDisplay,d=Ra(b,d,e,f),e=0;e<j.length;e++)b=a.aoData[j[e]]._aFilterData[c],d.test(b)&&g.push(j[e]);a.aiDisplay=g}}function yb(a,b,c,d,e,f){var d=Ra(b,d,e,f),f=a.oPreviousSearch.sSearch,g=a.aiDisplayMaster,j,e=[];0!==m.ext.search.length&&(c=!0);j=Bb(a);if(0>=b.length)a.aiDisplay=
+g.slice();else{if(j||c||f.length>b.length||0!==b.indexOf(f)||a.bSorted)a.aiDisplay=g.slice();b=a.aiDisplay;for(c=0;c<b.length;c++)d.test(a.aoData[b[c]]._sFilterRow)&&e.push(b[c]);a.aiDisplay=e}}function Ra(a,b,c,d){a=b?a:Sa(a);c&&(a="^(?=.*?"+h.map(a.match(/"[^"]+"|[^ ]+/g)||[""],function(a){if('"'===a.charAt(0))var b=a.match(/^"(.*)"$/),a=b?b[1]:a;return a.replace('"',"")}).join(")(?=.*?")+").*$");return RegExp(a,d?"i":"")}function Bb(a){var b=a.aoColumns,c,d,e,f,g,j,i,h,l=m.ext.type.search;c=!1;
+d=0;for(f=a.aoData.length;d<f;d++)if(h=a.aoData[d],!h._aFilterData){j=[];e=0;for(g=b.length;e<g;e++)c=b[e],c.bSearchable?(i=B(a,d,e,"filter"),l[c.sType]&&(i=l[c.sType](i)),null===i&&(i=""),"string"!==typeof i&&i.toString&&(i=i.toString())):i="",i.indexOf&&-1!==i.indexOf("&")&&(xa.innerHTML=i,i=$b?xa.textContent:xa.innerText),i.replace&&(i=i.replace(/[\r\n]/g,"")),j.push(i);h._aFilterData=j;h._sFilterRow=j.join(" ");c=!0}return c}function Cb(a){return{search:a.sSearch,smart:a.bSmart,regex:a.bRegex,
+caseInsensitive:a.bCaseInsensitive}}function Db(a){return{sSearch:a.search,bSmart:a.smart,bRegex:a.regex,bCaseInsensitive:a.caseInsensitive}}function ub(a){var b=a.sTableId,c=a.aanFeatures.i,d=h("<div/>",{"class":a.oClasses.sInfo,id:!c?b+"_info":null});c||(a.aoDrawCallback.push({fn:Eb,sName:"information"}),d.attr("role","status").attr("aria-live","polite"),h(a.nTable).attr("aria-describedby",b+"_info"));return d[0]}function Eb(a){var b=a.aanFeatures.i;if(0!==b.length){var c=a.oLanguage,d=a._iDisplayStart+
+1,e=a.fnDisplayEnd(),f=a.fnRecordsTotal(),g=a.fnRecordsDisplay(),j=g?c.sInfo:c.sInfoEmpty;g!==f&&(j+=" "+c.sInfoFiltered);j+=c.sInfoPostFix;j=Fb(a,j);c=c.fnInfoCallback;null!==c&&(j=c.call(a.oInstance,a,d,e,f,g,j));h(b).html(j)}}function Fb(a,b){var c=a.fnFormatNumber,d=a._iDisplayStart+1,e=a._iDisplayLength,f=a.fnRecordsDisplay(),g=-1===e;return b.replace(/_START_/g,c.call(a,d)).replace(/_END_/g,c.call(a,a.fnDisplayEnd())).replace(/_MAX_/g,c.call(a,a.fnRecordsTotal())).replace(/_TOTAL_/g,c.call(a,
+f)).replace(/_PAGE_/g,c.call(a,g?1:Math.ceil(d/e))).replace(/_PAGES_/g,c.call(a,g?1:Math.ceil(f/e)))}function ha(a){var b,c,d=a.iInitDisplayStart,e=a.aoColumns,f;c=a.oFeatures;var g=a.bDeferLoading;if(a.bInitialised){pb(a);mb(a);fa(a,a.aoHeader);fa(a,a.aoFooter);C(a,!0);c.bAutoWidth&&Ha(a);b=0;for(c=e.length;b<c;b++)f=e[b],f.sWidth&&(f.nTh.style.width=v(f.sWidth));s(a,null,"preInit",[a]);T(a);e=y(a);if("ssp"!=e||g)"ajax"==e?ua(a,[],function(c){var f=va(a,c);for(b=0;b<f.length;b++)N(a,f[b]);a.iInitDisplayStart=
+d;T(a);C(a,!1);wa(a,c)},a):(C(a,!1),wa(a))}else setTimeout(function(){ha(a)},200)}function wa(a,b){a._bInitComplete=!0;(b||a.oInit.aaData)&&Z(a);s(a,null,"plugin-init",[a,b]);s(a,"aoInitComplete","init",[a,b])}function Ta(a,b){var c=parseInt(b,10);a._iDisplayLength=c;Ua(a);s(a,null,"length",[a,c])}function qb(a){for(var b=a.oClasses,c=a.sTableId,d=a.aLengthMenu,e=h.isArray(d[0]),f=e?d[0]:d,d=e?d[1]:d,e=h("<select/>",{name:c+"_length","aria-controls":c,"class":b.sLengthSelect}),g=0,j=f.length;g<j;g++)e[0][g]=
+new Option(d[g],f[g]);var i=h("<div><label/></div>").addClass(b.sLength);a.aanFeatures.l||(i[0].id=c+"_length");i.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",e[0].outerHTML));h("select",i).val(a._iDisplayLength).on("change.DT",function(){Ta(a,h(this).val());O(a)});h(a.nTable).on("length.dt.DT",function(b,c,d){a===c&&h("select",i).val(d)});return i[0]}function vb(a){var b=a.sPaginationType,c=m.ext.pager[b],d="function"===typeof c,e=function(a){O(a)},b=h("<div/>").addClass(a.oClasses.sPaging+
+b)[0],f=a.aanFeatures;d||c.fnInit(a,b,e);f.p||(b.id=a.sTableId+"_paginate",a.aoDrawCallback.push({fn:function(a){if(d){var b=a._iDisplayStart,i=a._iDisplayLength,h=a.fnRecordsDisplay(),l=-1===i,b=l?0:Math.ceil(b/i),i=l?1:Math.ceil(h/i),h=c(b,i),k,l=0;for(k=f.p.length;l<k;l++)Pa(a,"pageButton")(a,f.p[l],l,h,b,i)}else c.fnUpdate(a,e)},sName:"pagination"}));return b}function Va(a,b,c){var d=a._iDisplayStart,e=a._iDisplayLength,f=a.fnRecordsDisplay();0===f||-1===e?d=0:"number"===typeof b?(d=b*e,d>f&&
+(d=0)):"first"==b?d=0:"previous"==b?(d=0<=e?d-e:0,0>d&&(d=0)):"next"==b?d+e<f&&(d+=e):"last"==b?d=Math.floor((f-1)/e)*e:K(a,0,"Unknown paging action: "+b,5);b=a._iDisplayStart!==d;a._iDisplayStart=d;b&&(s(a,null,"page",[a]),c&&O(a));return b}function sb(a){return h("<div/>",{id:!a.aanFeatures.r?a.sTableId+"_processing":null,"class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]}function C(a,b){a.oFeatures.bProcessing&&h(a.aanFeatures.r).css("display",b?"block":"none");
+s(a,null,"processing",[a,b])}function tb(a){var b=h(a.nTable);b.attr("role","grid");var c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var d=c.sX,e=c.sY,f=a.oClasses,g=b.children("caption"),j=g.length?g[0]._captionSide:null,i=h(b[0].cloneNode(!1)),n=h(b[0].cloneNode(!1)),l=b.children("tfoot");l.length||(l=null);i=h("<div/>",{"class":f.sScrollWrapper}).append(h("<div/>",{"class":f.sScrollHead}).css({overflow:"hidden",position:"relative",border:0,width:d?!d?null:v(d):"100%"}).append(h("<div/>",
+{"class":f.sScrollHeadInner}).css({"box-sizing":"content-box",width:c.sXInner||"100%"}).append(i.removeAttr("id").css("margin-left",0).append("top"===j?g:null).append(b.children("thead"))))).append(h("<div/>",{"class":f.sScrollBody}).css({position:"relative",overflow:"auto",width:!d?null:v(d)}).append(b));l&&i.append(h("<div/>",{"class":f.sScrollFoot}).css({overflow:"hidden",border:0,width:d?!d?null:v(d):"100%"}).append(h("<div/>",{"class":f.sScrollFootInner}).append(n.removeAttr("id").css("margin-left",
+0).append("bottom"===j?g:null).append(b.children("tfoot")))));var b=i.children(),k=b[0],f=b[1],r=l?b[2]:null;if(d)h(f).on("scroll.DT",function(){var a=this.scrollLeft;k.scrollLeft=a;l&&(r.scrollLeft=a)});h(f).css(e&&c.bCollapse?"max-height":"height",e);a.nScrollHead=k;a.nScrollBody=f;a.nScrollFoot=r;a.aoDrawCallback.push({fn:ma,sName:"scrolling"});return i[0]}function ma(a){var b=a.oScroll,c=b.sX,d=b.sXInner,e=b.sY,b=b.iBarWidth,f=h(a.nScrollHead),g=f[0].style,j=f.children("div"),i=j[0].style,n=j.children("table"),
+j=a.nScrollBody,l=h(j),q=j.style,r=h(a.nScrollFoot).children("div"),m=r.children("table"),p=h(a.nTHead),o=h(a.nTable),t=o[0],s=t.style,u=a.nTFoot?h(a.nTFoot):null,x=a.oBrowser,U=x.bScrollOversize,ac=D(a.aoColumns,"nTh"),P,L,Q,w,Wa=[],y=[],z=[],A=[],B,C=function(a){a=a.style;a.paddingTop="0";a.paddingBottom="0";a.borderTopWidth="0";a.borderBottomWidth="0";a.height=0};L=j.scrollHeight>j.clientHeight;if(a.scrollBarVis!==L&&a.scrollBarVis!==k)a.scrollBarVis=L,Z(a);else{a.scrollBarVis=L;o.children("thead, tfoot").remove();
+u&&(Q=u.clone().prependTo(o),P=u.find("tr"),Q=Q.find("tr"));w=p.clone().prependTo(o);p=p.find("tr");L=w.find("tr");w.find("th, td").removeAttr("tabindex");c||(q.width="100%",f[0].style.width="100%");h.each(ta(a,w),function(b,c){B=$(a,b);c.style.width=a.aoColumns[B].sWidth});u&&I(function(a){a.style.width=""},Q);f=o.outerWidth();if(""===c){s.width="100%";if(U&&(o.find("tbody").height()>j.offsetHeight||"scroll"==l.css("overflow-y")))s.width=v(o.outerWidth()-b);f=o.outerWidth()}else""!==d&&(s.width=
+v(d),f=o.outerWidth());I(C,L);I(function(a){z.push(a.innerHTML);Wa.push(v(h(a).css("width")))},L);I(function(a,b){if(h.inArray(a,ac)!==-1)a.style.width=Wa[b]},p);h(L).height(0);u&&(I(C,Q),I(function(a){A.push(a.innerHTML);y.push(v(h(a).css("width")))},Q),I(function(a,b){a.style.width=y[b]},P),h(Q).height(0));I(function(a,b){a.innerHTML='<div class="dataTables_sizing" style="height:0;overflow:hidden;">'+z[b]+"</div>";a.style.width=Wa[b]},L);u&&I(function(a,b){a.innerHTML='<div class="dataTables_sizing" style="height:0;overflow:hidden;">'+
+A[b]+"</div>";a.style.width=y[b]},Q);if(o.outerWidth()<f){P=j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")?f+b:f;if(U&&(j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")))s.width=v(P-b);(""===c||""!==d)&&K(a,1,"Possible column misalignment",6)}else P="100%";q.width=v(P);g.width=v(P);u&&(a.nScrollFoot.style.width=v(P));!e&&U&&(q.height=v(t.offsetHeight+b));c=o.outerWidth();n[0].style.width=v(c);i.width=v(c);d=o.height()>j.clientHeight||"scroll"==l.css("overflow-y");e="padding"+
+(x.bScrollbarLeft?"Left":"Right");i[e]=d?b+"px":"0px";u&&(m[0].style.width=v(c),r[0].style.width=v(c),r[0].style[e]=d?b+"px":"0px");o.children("colgroup").insertBefore(o.children("thead"));l.scroll();if((a.bSorted||a.bFiltered)&&!a._drawHold)j.scrollTop=0}}function I(a,b,c){for(var d=0,e=0,f=b.length,g,j;e<f;){g=b[e].firstChild;for(j=c?c[e].firstChild:null;g;)1===g.nodeType&&(c?a(g,j,d):a(g,d),d++),g=g.nextSibling,j=c?j.nextSibling:null;e++}}function Ha(a){var b=a.nTable,c=a.aoColumns,d=a.oScroll,
+e=d.sY,f=d.sX,g=d.sXInner,j=c.length,i=na(a,"bVisible"),n=h("th",a.nTHead),l=b.getAttribute("width"),k=b.parentNode,r=!1,m,p,o=a.oBrowser,d=o.bScrollOversize;(m=b.style.width)&&-1!==m.indexOf("%")&&(l=m);for(m=0;m<i.length;m++)p=c[i[m]],null!==p.sWidth&&(p.sWidth=Gb(p.sWidthOrig,k),r=!0);if(d||!r&&!f&&!e&&j==ba(a)&&j==n.length)for(m=0;m<j;m++)i=$(a,m),null!==i&&(c[i].sWidth=v(n.eq(m).width()));else{j=h(b).clone().css("visibility","hidden").removeAttr("id");j.find("tbody tr").remove();var t=h("<tr/>").appendTo(j.find("tbody"));
+j.find("thead, tfoot").remove();j.append(h(a.nTHead).clone()).append(h(a.nTFoot).clone());j.find("tfoot th, tfoot td").css("width","");n=ta(a,j.find("thead")[0]);for(m=0;m<i.length;m++)p=c[i[m]],n[m].style.width=null!==p.sWidthOrig&&""!==p.sWidthOrig?v(p.sWidthOrig):"",p.sWidthOrig&&f&&h(n[m]).append(h("<div/>").css({width:p.sWidthOrig,margin:0,padding:0,border:0,height:1}));if(a.aoData.length)for(m=0;m<i.length;m++)r=i[m],p=c[r],h(Hb(a,r)).clone(!1).append(p.sContentPadding).appendTo(t);h("[name]",
+j).removeAttr("name");p=h("<div/>").css(f||e?{position:"absolute",top:0,left:0,height:1,right:0,overflow:"hidden"}:{}).append(j).appendTo(k);f&&g?j.width(g):f?(j.css("width","auto"),j.removeAttr("width"),j.width()<k.clientWidth&&l&&j.width(k.clientWidth)):e?j.width(k.clientWidth):l&&j.width(l);for(m=e=0;m<i.length;m++)k=h(n[m]),g=k.outerWidth()-k.width(),k=o.bBounding?Math.ceil(n[m].getBoundingClientRect().width):k.outerWidth(),e+=k,c[i[m]].sWidth=v(k-g);b.style.width=v(e);p.remove()}l&&(b.style.width=
+v(l));if((l||f)&&!a._reszEvt)b=function(){h(E).on("resize.DT-"+a.sInstance,Qa(function(){Z(a)}))},d?setTimeout(b,1E3):b(),a._reszEvt=!0}function Gb(a,b){if(!a)return 0;var c=h("<div/>").css("width",v(a)).appendTo(b||H.body),d=c[0].offsetWidth;c.remove();return d}function Hb(a,b){var c=Ib(a,b);if(0>c)return null;var d=a.aoData[c];return!d.nTr?h("<td/>").html(B(a,c,b,"display"))[0]:d.anCells[b]}function Ib(a,b){for(var c,d=-1,e=-1,f=0,g=a.aoData.length;f<g;f++)c=B(a,f,b,"display")+"",c=c.replace(bc,
+""),c=c.replace(/ /g," "),c.length>d&&(d=c.length,e=f);return e}function v(a){return null===a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function W(a){var b,c,d=[],e=a.aoColumns,f,g,j,i;b=a.aaSortingFixed;c=h.isPlainObject(b);var n=[];f=function(a){a.length&&!h.isArray(a[0])?n.push(a):h.merge(n,a)};h.isArray(b)&&f(b);c&&b.pre&&f(b.pre);f(a.aaSorting);c&&b.post&&f(b.post);for(a=0;a<n.length;a++){i=n[a][0];f=e[i].aDataSort;b=0;for(c=f.length;b<c;b++)g=f[b],j=e[g].sType||
+"string",n[a]._idx===k&&(n[a]._idx=h.inArray(n[a][1],e[g].asSorting)),d.push({src:i,col:g,dir:n[a][1],index:n[a]._idx,type:j,formatter:m.ext.type.order[j+"-pre"]})}return d}function ob(a){var b,c,d=[],e=m.ext.type.order,f=a.aoData,g=0,j,i=a.aiDisplayMaster,h;Ia(a);h=W(a);b=0;for(c=h.length;b<c;b++)j=h[b],j.formatter&&g++,Jb(a,j.col);if("ssp"!=y(a)&&0!==h.length){b=0;for(c=i.length;b<c;b++)d[i[b]]=b;g===h.length?i.sort(function(a,b){var c,e,g,j,i=h.length,k=f[a]._aSortData,m=f[b]._aSortData;for(g=
+0;g<i;g++)if(j=h[g],c=k[j.col],e=m[j.col],c=c<e?-1:c>e?1:0,0!==c)return"asc"===j.dir?c:-c;c=d[a];e=d[b];return c<e?-1:c>e?1:0}):i.sort(function(a,b){var c,g,j,i,k=h.length,m=f[a]._aSortData,p=f[b]._aSortData;for(j=0;j<k;j++)if(i=h[j],c=m[i.col],g=p[i.col],i=e[i.type+"-"+i.dir]||e["string-"+i.dir],c=i(c,g),0!==c)return c;c=d[a];g=d[b];return c<g?-1:c>g?1:0})}a.bSorted=!0}function Kb(a){for(var b,c,d=a.aoColumns,e=W(a),a=a.oLanguage.oAria,f=0,g=d.length;f<g;f++){c=d[f];var j=c.asSorting;b=c.sTitle.replace(/<.*?>/g,
+"");var i=c.nTh;i.removeAttribute("aria-sort");c.bSortable&&(0<e.length&&e[0].col==f?(i.setAttribute("aria-sort","asc"==e[0].dir?"ascending":"descending"),c=j[e[0].index+1]||j[0]):c=j[0],b+="asc"===c?a.sSortAscending:a.sSortDescending);i.setAttribute("aria-label",b)}}function Xa(a,b,c,d){var e=a.aaSorting,f=a.aoColumns[b].asSorting,g=function(a,b){var c=a._idx;c===k&&(c=h.inArray(a[1],f));return c+1<f.length?c+1:b?null:0};"number"===typeof e[0]&&(e=a.aaSorting=[e]);c&&a.oFeatures.bSortMulti?(c=h.inArray(b,
+D(e,"0")),-1!==c?(b=g(e[c],!0),null===b&&1===e.length&&(b=0),null===b?e.splice(c,1):(e[c][1]=f[b],e[c]._idx=b)):(e.push([b,f[0],0]),e[e.length-1]._idx=0)):e.length&&e[0][0]==b?(b=g(e[0]),e.length=1,e[0][1]=f[b],e[0]._idx=b):(e.length=0,e.push([b,f[0]]),e[0]._idx=0);T(a);"function"==typeof d&&d(a)}function Oa(a,b,c,d){var e=a.aoColumns[c];Ya(b,{},function(b){!1!==e.bSortable&&(a.oFeatures.bProcessing?(C(a,!0),setTimeout(function(){Xa(a,c,b.shiftKey,d);"ssp"!==y(a)&&C(a,!1)},0)):Xa(a,c,b.shiftKey,d))})}
+function ya(a){var b=a.aLastSort,c=a.oClasses.sSortColumn,d=W(a),e=a.oFeatures,f,g;if(e.bSort&&e.bSortClasses){e=0;for(f=b.length;e<f;e++)g=b[e].src,h(D(a.aoData,"anCells",g)).removeClass(c+(2>e?e+1:3));e=0;for(f=d.length;e<f;e++)g=d[e].src,h(D(a.aoData,"anCells",g)).addClass(c+(2>e?e+1:3))}a.aLastSort=d}function Jb(a,b){var c=a.aoColumns[b],d=m.ext.order[c.sSortDataType],e;d&&(e=d.call(a.oInstance,a,b,aa(a,b)));for(var f,g=m.ext.type.order[c.sType+"-pre"],j=0,i=a.aoData.length;j<i;j++)if(c=a.aoData[j],
+c._aSortData||(c._aSortData=[]),!c._aSortData[b]||d)f=d?e[j]:B(a,j,b,"sort"),c._aSortData[b]=g?g(f):f}function za(a){if(a.oFeatures.bStateSave&&!a.bDestroying){var b={time:+new Date,start:a._iDisplayStart,length:a._iDisplayLength,order:h.extend(!0,[],a.aaSorting),search:Cb(a.oPreviousSearch),columns:h.map(a.aoColumns,function(b,d){return{visible:b.bVisible,search:Cb(a.aoPreSearchCols[d])}})};s(a,"aoStateSaveParams","stateSaveParams",[a,b]);a.oSavedState=b;a.fnStateSaveCallback.call(a.oInstance,a,
+b)}}function Lb(a,b,c){var d,e,f=a.aoColumns,b=function(b){if(b&&b.time){var g=s(a,"aoStateLoadParams","stateLoadParams",[a,b]);if(-1===h.inArray(!1,g)&&(g=a.iStateDuration,!(0<g&&b.time<+new Date-1E3*g)&&!(b.columns&&f.length!==b.columns.length))){a.oLoadedState=h.extend(!0,{},b);b.start!==k&&(a._iDisplayStart=b.start,a.iInitDisplayStart=b.start);b.length!==k&&(a._iDisplayLength=b.length);b.order!==k&&(a.aaSorting=[],h.each(b.order,function(b,c){a.aaSorting.push(c[0]>=f.length?[0,c[1]]:c)}));b.search!==
+k&&h.extend(a.oPreviousSearch,Db(b.search));if(b.columns){d=0;for(e=b.columns.length;d<e;d++)g=b.columns[d],g.visible!==k&&(f[d].bVisible=g.visible),g.search!==k&&h.extend(a.aoPreSearchCols[d],Db(g.search))}s(a,"aoStateLoaded","stateLoaded",[a,b])}}c()};if(a.oFeatures.bStateSave){var g=a.fnStateLoadCallback.call(a.oInstance,a,b);g!==k&&b(g)}else c()}function Aa(a){var b=m.settings,a=h.inArray(a,D(b,"nTable"));return-1!==a?b[a]:null}function K(a,b,c,d){c="DataTables warning: "+(a?"table id="+a.sTableId+
+" - ":"")+c;d&&(c+=". For more information about this error, please see http://datatables.net/tn/"+d);if(b)E.console&&console.log&&console.log(c);else if(b=m.ext,b=b.sErrMode||b.errMode,a&&s(a,null,"error",[a,d,c]),"alert"==b)alert(c);else{if("throw"==b)throw Error(c);"function"==typeof b&&b(a,d,c)}}function F(a,b,c,d){h.isArray(c)?h.each(c,function(c,d){h.isArray(d)?F(a,b,d[0],d[1]):F(a,b,d)}):(d===k&&(d=c),b[c]!==k&&(a[d]=b[c]))}function Mb(a,b,c){var d,e;for(e in b)b.hasOwnProperty(e)&&(d=b[e],
+h.isPlainObject(d)?(h.isPlainObject(a[e])||(a[e]={}),h.extend(!0,a[e],d)):a[e]=c&&"data"!==e&&"aaData"!==e&&h.isArray(d)?d.slice():d);return a}function Ya(a,b,c){h(a).on("click.DT",b,function(b){a.blur();c(b)}).on("keypress.DT",b,function(a){13===a.which&&(a.preventDefault(),c(a))}).on("selectstart.DT",function(){return!1})}function z(a,b,c,d){c&&a[b].push({fn:c,sName:d})}function s(a,b,c,d){var e=[];b&&(e=h.map(a[b].slice().reverse(),function(b){return b.fn.apply(a.oInstance,d)}));null!==c&&(b=h.Event(c+
+".dt"),h(a.nTable).trigger(b,d),e.push(b.result));return e}function Ua(a){var b=a._iDisplayStart,c=a.fnDisplayEnd(),d=a._iDisplayLength;b>=c&&(b=c-d);b-=b%d;if(-1===d||0>b)b=0;a._iDisplayStart=b}function Pa(a,b){var c=a.renderer,d=m.ext.renderer[b];return h.isPlainObject(c)&&c[b]?d[c[b]]||d._:"string"===typeof c?d[c]||d._:d._}function y(a){return a.oFeatures.bServerSide?"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function ia(a,b){var c=[],c=Nb.numbers_length,d=Math.floor(c/2);b<=c?c=X(0,b):a<=d?(c=X(0,
+c-2),c.push("ellipsis"),c.push(b-1)):(a>=b-1-d?c=X(b-(c-2),b):(c=X(a-d+2,a+d-1),c.push("ellipsis"),c.push(b-1)),c.splice(0,0,"ellipsis"),c.splice(0,0,0));c.DT_el="span";return c}function fb(a){h.each({num:function(b){return Ba(b,a)},"num-fmt":function(b){return Ba(b,a,Za)},"html-num":function(b){return Ba(b,a,Ca)},"html-num-fmt":function(b){return Ba(b,a,Ca,Za)}},function(b,c){x.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(x.type.search[b+a]=x.type.search.html)})}function Ob(a){return function(){var b=
+[Aa(this[m.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return m.ext.internal[a].apply(this,b)}}var m=function(a){this.$=function(a,b){return this.api(!0).$(a,b)};this._=function(a,b){return this.api(!0).rows(a,b).data()};this.api=function(a){return a?new t(Aa(this[x.iApiIndex])):new t(this)};this.fnAddData=function(a,b){var c=this.api(!0),d=h.isArray(a)&&(h.isArray(a[0])||h.isPlainObject(a[0]))?c.rows.add(a):c.row.add(a);(b===k||b)&&c.draw();return d.flatten().toArray()};this.fnAdjustColumnSizing=
+function(a){var b=this.api(!0).columns.adjust(),c=b.settings()[0],d=c.oScroll;a===k||a?b.draw(!1):(""!==d.sX||""!==d.sY)&&ma(c)};this.fnClearTable=function(a){var b=this.api(!0).clear();(a===k||a)&&b.draw()};this.fnClose=function(a){this.api(!0).row(a).child.hide()};this.fnDeleteRow=function(a,b,c){var d=this.api(!0),a=d.rows(a),e=a.settings()[0],h=e.aoData[a[0][0]];a.remove();b&&b.call(this,e,h);(c===k||c)&&d.draw();return h};this.fnDestroy=function(a){this.api(!0).destroy(a)};this.fnDraw=function(a){this.api(!0).draw(a)};
+this.fnFilter=function(a,b,c,d,e,h){e=this.api(!0);null===b||b===k?e.search(a,c,d,h):e.column(b).search(a,c,d,h);e.draw()};this.fnGetData=function(a,b){var c=this.api(!0);if(a!==k){var d=a.nodeName?a.nodeName.toLowerCase():"";return b!==k||"td"==d||"th"==d?c.cell(a,b).data():c.row(a).data()||null}return c.data().toArray()};this.fnGetNodes=function(a){var b=this.api(!0);return a!==k?b.row(a).node():b.rows().nodes().flatten().toArray()};this.fnGetPosition=function(a){var b=this.api(!0),c=a.nodeName.toUpperCase();
+return"TR"==c?b.row(a).index():"TD"==c||"TH"==c?(a=b.cell(a).index(),[a.row,a.columnVisible,a.column]):null};this.fnIsOpen=function(a){return this.api(!0).row(a).child.isShown()};this.fnOpen=function(a,b,c){return this.api(!0).row(a).child(b,c).show().child()[0]};this.fnPageChange=function(a,b){var c=this.api(!0).page(a);(b===k||b)&&c.draw(!1)};this.fnSetColumnVis=function(a,b,c){a=this.api(!0).column(a).visible(b);(c===k||c)&&a.columns.adjust().draw()};this.fnSettings=function(){return Aa(this[x.iApiIndex])};
+this.fnSort=function(a){this.api(!0).order(a).draw()};this.fnSortListener=function(a,b,c){this.api(!0).order.listener(a,b,c)};this.fnUpdate=function(a,b,c,d,e){var h=this.api(!0);c===k||null===c?h.row(b).data(a):h.cell(b,c).data(a);(e===k||e)&&h.columns.adjust();(d===k||d)&&h.draw();return 0};this.fnVersionCheck=x.fnVersionCheck;var b=this,c=a===k,d=this.length;c&&(a={});this.oApi=this.internal=x.internal;for(var e in m.ext.internal)e&&(this[e]=Ob(e));this.each(function(){var e={},g=1<d?Mb(e,a,!0):
+a,j=0,i,e=this.getAttribute("id"),n=!1,l=m.defaults,q=h(this);if("table"!=this.nodeName.toLowerCase())K(null,0,"Non-table node initialisation ("+this.nodeName+")",2);else{gb(l);hb(l.column);J(l,l,!0);J(l.column,l.column,!0);J(l,h.extend(g,q.data()));var r=m.settings,j=0;for(i=r.length;j<i;j++){var p=r[j];if(p.nTable==this||p.nTHead.parentNode==this||p.nTFoot&&p.nTFoot.parentNode==this){var t=g.bRetrieve!==k?g.bRetrieve:l.bRetrieve;if(c||t)return p.oInstance;if(g.bDestroy!==k?g.bDestroy:l.bDestroy){p.oInstance.fnDestroy();
+break}else{K(p,0,"Cannot reinitialise DataTable",3);return}}if(p.sTableId==this.id){r.splice(j,1);break}}if(null===e||""===e)this.id=e="DataTables_Table_"+m.ext._unique++;var o=h.extend(!0,{},m.models.oSettings,{sDestroyWidth:q[0].style.width,sInstance:e,sTableId:e});o.nTable=this;o.oApi=b.internal;o.oInit=g;r.push(o);o.oInstance=1===b.length?b:q.dataTable();gb(g);g.oLanguage&&Fa(g.oLanguage);g.aLengthMenu&&!g.iDisplayLength&&(g.iDisplayLength=h.isArray(g.aLengthMenu[0])?g.aLengthMenu[0][0]:g.aLengthMenu[0]);
+g=Mb(h.extend(!0,{},l),g);F(o.oFeatures,g,"bPaginate bLengthChange bFilter bSort bSortMulti bInfo bProcessing bAutoWidth bSortClasses bServerSide bDeferRender".split(" "));F(o,g,["asStripeClasses","ajax","fnServerData","fnFormatNumber","sServerMethod","aaSorting","aaSortingFixed","aLengthMenu","sPaginationType","sAjaxSource","sAjaxDataProp","iStateDuration","sDom","bSortCellsTop","iTabIndex","fnStateLoadCallback","fnStateSaveCallback","renderer","searchDelay","rowId",["iCookieDuration","iStateDuration"],
+["oSearch","oPreviousSearch"],["aoSearchCols","aoPreSearchCols"],["iDisplayLength","_iDisplayLength"],["bJQueryUI","bJUI"]]);F(o.oScroll,g,[["sScrollX","sX"],["sScrollXInner","sXInner"],["sScrollY","sY"],["bScrollCollapse","bCollapse"]]);F(o.oLanguage,g,"fnInfoCallback");z(o,"aoDrawCallback",g.fnDrawCallback,"user");z(o,"aoServerParams",g.fnServerParams,"user");z(o,"aoStateSaveParams",g.fnStateSaveParams,"user");z(o,"aoStateLoadParams",g.fnStateLoadParams,"user");z(o,"aoStateLoaded",g.fnStateLoaded,
+"user");z(o,"aoRowCallback",g.fnRowCallback,"user");z(o,"aoRowCreatedCallback",g.fnCreatedRow,"user");z(o,"aoHeaderCallback",g.fnHeaderCallback,"user");z(o,"aoFooterCallback",g.fnFooterCallback,"user");z(o,"aoInitComplete",g.fnInitComplete,"user");z(o,"aoPreDrawCallback",g.fnPreDrawCallback,"user");o.rowIdFn=R(g.rowId);ib(o);var u=o.oClasses;g.bJQueryUI?(h.extend(u,m.ext.oJUIClasses,g.oClasses),g.sDom===l.sDom&&"lfrtip"===l.sDom&&(o.sDom='<"H"lfr>t<"F"ip>'),o.renderer)?h.isPlainObject(o.renderer)&&
+!o.renderer.header&&(o.renderer.header="jqueryui"):o.renderer="jqueryui":h.extend(u,m.ext.classes,g.oClasses);q.addClass(u.sTable);o.iInitDisplayStart===k&&(o.iInitDisplayStart=g.iDisplayStart,o._iDisplayStart=g.iDisplayStart);null!==g.iDeferLoading&&(o.bDeferLoading=!0,e=h.isArray(g.iDeferLoading),o._iRecordsDisplay=e?g.iDeferLoading[0]:g.iDeferLoading,o._iRecordsTotal=e?g.iDeferLoading[1]:g.iDeferLoading);var v=o.oLanguage;h.extend(!0,v,g.oLanguage);v.sUrl&&(h.ajax({dataType:"json",url:v.sUrl,success:function(a){Fa(a);
+J(l.oLanguage,a);h.extend(true,v,a);ha(o)},error:function(){ha(o)}}),n=!0);null===g.asStripeClasses&&(o.asStripeClasses=[u.sStripeOdd,u.sStripeEven]);var e=o.asStripeClasses,x=q.children("tbody").find("tr").eq(0);-1!==h.inArray(!0,h.map(e,function(a){return x.hasClass(a)}))&&(h("tbody tr",this).removeClass(e.join(" ")),o.asDestroyStripes=e.slice());e=[];r=this.getElementsByTagName("thead");0!==r.length&&(ea(o.aoHeader,r[0]),e=ta(o));if(null===g.aoColumns){r=[];j=0;for(i=e.length;j<i;j++)r.push(null)}else r=
+g.aoColumns;j=0;for(i=r.length;j<i;j++)Ga(o,e?e[j]:null);kb(o,g.aoColumnDefs,r,function(a,b){la(o,a,b)});if(x.length){var w=function(a,b){return a.getAttribute("data-"+b)!==null?b:null};h(x[0]).children("th, td").each(function(a,b){var c=o.aoColumns[a];if(c.mData===a){var d=w(b,"sort")||w(b,"order"),e=w(b,"filter")||w(b,"search");if(d!==null||e!==null){c.mData={_:a+".display",sort:d!==null?a+".@data-"+d:k,type:d!==null?a+".@data-"+d:k,filter:e!==null?a+".@data-"+e:k};la(o,a)}}})}var U=o.oFeatures,
+e=function(){if(g.aaSorting===k){var a=o.aaSorting;j=0;for(i=a.length;j<i;j++)a[j][1]=o.aoColumns[j].asSorting[0]}ya(o);U.bSort&&z(o,"aoDrawCallback",function(){if(o.bSorted){var a=W(o),b={};h.each(a,function(a,c){b[c.src]=c.dir});s(o,null,"order",[o,a,b]);Kb(o)}});z(o,"aoDrawCallback",function(){(o.bSorted||y(o)==="ssp"||U.bDeferRender)&&ya(o)},"sc");var a=q.children("caption").each(function(){this._captionSide=h(this).css("caption-side")}),b=q.children("thead");b.length===0&&(b=h("<thead/>").appendTo(q));
+o.nTHead=b[0];b=q.children("tbody");b.length===0&&(b=h("<tbody/>").appendTo(q));o.nTBody=b[0];b=q.children("tfoot");if(b.length===0&&a.length>0&&(o.oScroll.sX!==""||o.oScroll.sY!==""))b=h("<tfoot/>").appendTo(q);if(b.length===0||b.children().length===0)q.addClass(u.sNoFooter);else if(b.length>0){o.nTFoot=b[0];ea(o.aoFooter,o.nTFoot)}if(g.aaData)for(j=0;j<g.aaData.length;j++)N(o,g.aaData[j]);else(o.bDeferLoading||y(o)=="dom")&&oa(o,h(o.nTBody).children("tr"));o.aiDisplay=o.aiDisplayMaster.slice();
+o.bInitialised=true;n===false&&ha(o)};g.bStateSave?(U.bStateSave=!0,z(o,"aoDrawCallback",za,"state_save"),Lb(o,g,e)):e()}});b=null;return this},x,t,p,u,$a={},Pb=/[\r\n]/g,Ca=/<.*?>/g,cc=/^\d{2,4}[\.\/\-]\d{1,2}[\.\/\-]\d{1,2}([T ]{1}\d{1,2}[:\.]\d{2}([\.:]\d{2})?)?$/,dc=RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),Za=/[',$£€¥%\u2009\u202F\u20BD\u20a9\u20BArfk]/gi,M=function(a){return!a||!0===a||"-"===a?!0:!1},Qb=function(a){var b=parseInt(a,10);return!isNaN(b)&&
+isFinite(a)?b:null},Rb=function(a,b){$a[b]||($a[b]=RegExp(Sa(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace($a[b],"."):a},ab=function(a,b,c){var d="string"===typeof a;if(M(a))return!0;b&&d&&(a=Rb(a,b));c&&d&&(a=a.replace(Za,""));return!isNaN(parseFloat(a))&&isFinite(a)},Sb=function(a,b,c){return M(a)?!0:!(M(a)||"string"===typeof a)?null:ab(a.replace(Ca,""),b,c)?!0:null},D=function(a,b,c){var d=[],e=0,f=a.length;if(c!==k)for(;e<f;e++)a[e]&&a[e][b]&&d.push(a[e][b][c]);else for(;e<
+f;e++)a[e]&&d.push(a[e][b]);return d},ja=function(a,b,c,d){var e=[],f=0,g=b.length;if(d!==k)for(;f<g;f++)a[b[f]][c]&&e.push(a[b[f]][c][d]);else for(;f<g;f++)e.push(a[b[f]][c]);return e},X=function(a,b){var c=[],d;b===k?(b=0,d=a):(d=b,b=a);for(var e=b;e<d;e++)c.push(e);return c},Tb=function(a){for(var b=[],c=0,d=a.length;c<d;c++)a[c]&&b.push(a[c]);return b},sa=function(a){var b;a:{if(!(2>a.length)){b=a.slice().sort();for(var c=b[0],d=1,e=b.length;d<e;d++){if(b[d]===c){b=!1;break a}c=b[d]}}b=!0}if(b)return a.slice();
+b=[];var e=a.length,f,g=0,d=0;a:for(;d<e;d++){c=a[d];for(f=0;f<g;f++)if(b[f]===c)continue a;b.push(c);g++}return b};m.util={throttle:function(a,b){var c=b!==k?b:200,d,e;return function(){var b=this,g=+new Date,h=arguments;d&&g<d+c?(clearTimeout(e),e=setTimeout(function(){d=k;a.apply(b,h)},c)):(d=g,a.apply(b,h))}},escapeRegex:function(a){return a.replace(dc,"\\$1")}};var A=function(a,b,c){a[b]!==k&&(a[c]=a[b])},ca=/\[.*?\]$/,V=/\(\)$/,Sa=m.util.escapeRegex,xa=h("<div>")[0],$b=xa.textContent!==k,bc=
+/<.*?>/g,Qa=m.util.throttle,Ub=[],w=Array.prototype,ec=function(a){var b,c,d=m.settings,e=h.map(d,function(a){return a.nTable});if(a){if(a.nTable&&a.oApi)return[a];if(a.nodeName&&"table"===a.nodeName.toLowerCase())return b=h.inArray(a,e),-1!==b?[d[b]]:null;if(a&&"function"===typeof a.settings)return a.settings().toArray();"string"===typeof a?c=h(a):a instanceof h&&(c=a)}else return[];if(c)return c.map(function(){b=h.inArray(this,e);return-1!==b?d[b]:null}).toArray()};t=function(a,b){if(!(this instanceof
+t))return new t(a,b);var c=[],d=function(a){(a=ec(a))&&(c=c.concat(a))};if(h.isArray(a))for(var e=0,f=a.length;e<f;e++)d(a[e]);else d(a);this.context=sa(c);b&&h.merge(this,b);this.selector={rows:null,cols:null,opts:null};t.extend(this,this,Ub)};m.Api=t;h.extend(t.prototype,{any:function(){return 0!==this.count()},concat:w.concat,context:[],count:function(){return this.flatten().length},each:function(a){for(var b=0,c=this.length;b<c;b++)a.call(this,this[b],b,this);return this},eq:function(a){var b=
+this.context;return b.length>a?new t(b[a],this[a]):null},filter:function(a){var b=[];if(w.filter)b=w.filter.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)a.call(this,this[c],c,this)&&b.push(this[c]);return new t(this.context,b)},flatten:function(){var a=[];return new t(this.context,a.concat.apply(a,this.toArray()))},join:w.join,indexOf:w.indexOf||function(a,b){for(var c=b||0,d=this.length;c<d;c++)if(this[c]===a)return c;return-1},iterator:function(a,b,c,d){var e=[],f,g,h,i,n,l=this.context,
+m,p,u=this.selector;"string"===typeof a&&(d=c,c=b,b=a,a=!1);g=0;for(h=l.length;g<h;g++){var s=new t(l[g]);if("table"===b)f=c.call(s,l[g],g),f!==k&&e.push(f);else if("columns"===b||"rows"===b)f=c.call(s,l[g],this[g],g),f!==k&&e.push(f);else if("column"===b||"column-rows"===b||"row"===b||"cell"===b){p=this[g];"column-rows"===b&&(m=Da(l[g],u.opts));i=0;for(n=p.length;i<n;i++)f=p[i],f="cell"===b?c.call(s,l[g],f.row,f.column,g,i):c.call(s,l[g],f,g,i,m),f!==k&&e.push(f)}}return e.length||d?(a=new t(l,a?
+e.concat.apply([],e):e),b=a.selector,b.rows=u.rows,b.cols=u.cols,b.opts=u.opts,a):this},lastIndexOf:w.lastIndexOf||function(a,b){return this.indexOf.apply(this.toArray.reverse(),arguments)},length:0,map:function(a){var b=[];if(w.map)b=w.map.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)b.push(a.call(this,this[c],c));return new t(this.context,b)},pluck:function(a){return this.map(function(b){return b[a]})},pop:w.pop,push:w.push,reduce:w.reduce||function(a,b){return jb(this,a,b,0,this.length,
+1)},reduceRight:w.reduceRight||function(a,b){return jb(this,a,b,this.length-1,-1,-1)},reverse:w.reverse,selector:null,shift:w.shift,slice:function(){return new t(this.context,this)},sort:w.sort,splice:w.splice,toArray:function(){return w.slice.call(this)},to$:function(){return h(this)},toJQuery:function(){return h(this)},unique:function(){return new t(this.context,sa(this))},unshift:w.unshift});t.extend=function(a,b,c){if(c.length&&b&&(b instanceof t||b.__dt_wrapper)){var d,e,f,g=function(a,b,c){return function(){var d=
+b.apply(a,arguments);t.extend(d,d,c.methodExt);return d}};d=0;for(e=c.length;d<e;d++)f=c[d],b[f.name]="function"===typeof f.val?g(a,f.val,f):h.isPlainObject(f.val)?{}:f.val,b[f.name].__dt_wrapper=!0,t.extend(a,b[f.name],f.propExt)}};t.register=p=function(a,b){if(h.isArray(a))for(var c=0,d=a.length;c<d;c++)t.register(a[c],b);else for(var e=a.split("."),f=Ub,g,j,c=0,d=e.length;c<d;c++){g=(j=-1!==e[c].indexOf("()"))?e[c].replace("()",""):e[c];var i;a:{i=0;for(var n=f.length;i<n;i++)if(f[i].name===g){i=
+f[i];break a}i=null}i||(i={name:g,val:{},methodExt:[],propExt:[]},f.push(i));c===d-1?i.val=b:f=j?i.methodExt:i.propExt}};t.registerPlural=u=function(a,b,c){t.register(a,c);t.register(b,function(){var a=c.apply(this,arguments);return a===this?this:a instanceof t?a.length?h.isArray(a[0])?new t(a.context,a[0]):a[0]:k:a})};p("tables()",function(a){var b;if(a){b=t;var c=this.context;if("number"===typeof a)a=[c[a]];else var d=h.map(c,function(a){return a.nTable}),a=h(d).filter(a).map(function(){var a=h.inArray(this,
+d);return c[a]}).toArray();b=new b(a)}else b=this;return b});p("table()",function(a){var a=this.tables(a),b=a.context;return b.length?new t(b[0]):a});u("tables().nodes()","table().node()",function(){return this.iterator("table",function(a){return a.nTable},1)});u("tables().body()","table().body()",function(){return this.iterator("table",function(a){return a.nTBody},1)});u("tables().header()","table().header()",function(){return this.iterator("table",function(a){return a.nTHead},1)});u("tables().footer()",
+"table().footer()",function(){return this.iterator("table",function(a){return a.nTFoot},1)});u("tables().containers()","table().container()",function(){return this.iterator("table",function(a){return a.nTableWrapper},1)});p("draw()",function(a){return this.iterator("table",function(b){"page"===a?O(b):("string"===typeof a&&(a="full-hold"===a?!1:!0),T(b,!1===a))})});p("page()",function(a){return a===k?this.page.info().page:this.iterator("table",function(b){Va(b,a)})});p("page.info()",function(){if(0===
+this.context.length)return k;var a=this.context[0],b=a._iDisplayStart,c=a.oFeatures.bPaginate?a._iDisplayLength:-1,d=a.fnRecordsDisplay(),e=-1===c;return{page:e?0:Math.floor(b/c),pages:e?1:Math.ceil(d/c),start:b,end:a.fnDisplayEnd(),length:c,recordsTotal:a.fnRecordsTotal(),recordsDisplay:d,serverSide:"ssp"===y(a)}});p("page.len()",function(a){return a===k?0!==this.context.length?this.context[0]._iDisplayLength:k:this.iterator("table",function(b){Ta(b,a)})});var Vb=function(a,b,c){if(c){var d=new t(a);
+d.one("draw",function(){c(d.ajax.json())})}if("ssp"==y(a))T(a,b);else{C(a,!0);var e=a.jqXHR;e&&4!==e.readyState&&e.abort();ua(a,[],function(c){pa(a);for(var c=va(a,c),d=0,e=c.length;d<e;d++)N(a,c[d]);T(a,b);C(a,!1)})}};p("ajax.json()",function(){var a=this.context;if(0<a.length)return a[0].json});p("ajax.params()",function(){var a=this.context;if(0<a.length)return a[0].oAjaxData});p("ajax.reload()",function(a,b){return this.iterator("table",function(c){Vb(c,!1===b,a)})});p("ajax.url()",function(a){var b=
+this.context;if(a===k){if(0===b.length)return k;b=b[0];return b.ajax?h.isPlainObject(b.ajax)?b.ajax.url:b.ajax:b.sAjaxSource}return this.iterator("table",function(b){h.isPlainObject(b.ajax)?b.ajax.url=a:b.ajax=a})});p("ajax.url().load()",function(a,b){return this.iterator("table",function(c){Vb(c,!1===b,a)})});var bb=function(a,b,c,d,e){var f=[],g,j,i,n,l,m;i=typeof b;if(!b||"string"===i||"function"===i||b.length===k)b=[b];i=0;for(n=b.length;i<n;i++){j=b[i]&&b[i].split&&!b[i].match(/[\[\(:]/)?b[i].split(","):
+[b[i]];l=0;for(m=j.length;l<m;l++)(g=c("string"===typeof j[l]?h.trim(j[l]):j[l]))&&g.length&&(f=f.concat(g))}a=x.selector[a];if(a.length){i=0;for(n=a.length;i<n;i++)f=a[i](d,e,f)}return sa(f)},cb=function(a){a||(a={});a.filter&&a.search===k&&(a.search=a.filter);return h.extend({search:"none",order:"current",page:"all"},a)},db=function(a){for(var b=0,c=a.length;b<c;b++)if(0<a[b].length)return a[0]=a[b],a[0].length=1,a.length=1,a.context=[a.context[b]],a;a.length=0;return a},Da=function(a,b){var c,
+d,e,f=[],g=a.aiDisplay;c=a.aiDisplayMaster;var j=b.search;d=b.order;e=b.page;if("ssp"==y(a))return"removed"===j?[]:X(0,c.length);if("current"==e){c=a._iDisplayStart;for(d=a.fnDisplayEnd();c<d;c++)f.push(g[c])}else if("current"==d||"applied"==d)f="none"==j?c.slice():"applied"==j?g.slice():h.map(c,function(a){return-1===h.inArray(a,g)?a:null});else if("index"==d||"original"==d){c=0;for(d=a.aoData.length;c<d;c++)"none"==j?f.push(c):(e=h.inArray(c,g),(-1===e&&"removed"==j||0<=e&&"applied"==j)&&f.push(c))}return f};
+p("rows()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=cb(b),c=this.iterator("table",function(c){var e=b,f;return bb("row",a,function(a){var b=Qb(a);if(b!==null&&!e)return[b];f||(f=Da(c,e));if(b!==null&&h.inArray(b,f)!==-1)return[b];if(a===null||a===k||a==="")return f;if(typeof a==="function")return h.map(f,function(b){var e=c.aoData[b];return a(b,e._aData,e.nTr)?b:null});b=Tb(ja(c.aoData,f,"nTr"));if(a.nodeName){if(a._DT_RowIndex!==k)return[a._DT_RowIndex];if(a._DT_CellIndex)return[a._DT_CellIndex.row];
+b=h(a).closest("*[data-dt-row]");return b.length?[b.data("dt-row")]:[]}if(typeof a==="string"&&a.charAt(0)==="#"){var i=c.aIds[a.replace(/^#/,"")];if(i!==k)return[i.idx]}return h(b).filter(a).map(function(){return this._DT_RowIndex}).toArray()},c,e)},1);c.selector.rows=a;c.selector.opts=b;return c});p("rows().nodes()",function(){return this.iterator("row",function(a,b){return a.aoData[b].nTr||k},1)});p("rows().data()",function(){return this.iterator(!0,"rows",function(a,b){return ja(a.aoData,b,"_aData")},
+1)});u("rows().cache()","row().cache()",function(a){return this.iterator("row",function(b,c){var d=b.aoData[c];return"search"===a?d._aFilterData:d._aSortData},1)});u("rows().invalidate()","row().invalidate()",function(a){return this.iterator("row",function(b,c){da(b,c,a)})});u("rows().indexes()","row().index()",function(){return this.iterator("row",function(a,b){return b},1)});u("rows().ids()","row().id()",function(a){for(var b=[],c=this.context,d=0,e=c.length;d<e;d++)for(var f=0,g=this[d].length;f<
+g;f++){var h=c[d].rowIdFn(c[d].aoData[this[d][f]]._aData);b.push((!0===a?"#":"")+h)}return new t(c,b)});u("rows().remove()","row().remove()",function(){var a=this;this.iterator("row",function(b,c,d){var e=b.aoData,f=e[c],g,h,i,n,l;e.splice(c,1);g=0;for(h=e.length;g<h;g++)if(i=e[g],l=i.anCells,null!==i.nTr&&(i.nTr._DT_RowIndex=g),null!==l){i=0;for(n=l.length;i<n;i++)l[i]._DT_CellIndex.row=g}qa(b.aiDisplayMaster,c);qa(b.aiDisplay,c);qa(a[d],c,!1);Ua(b);c=b.rowIdFn(f._aData);c!==k&&delete b.aIds[c]});
+this.iterator("table",function(a){for(var c=0,d=a.aoData.length;c<d;c++)a.aoData[c].idx=c});return this});p("rows.add()",function(a){var b=this.iterator("table",function(b){var c,f,g,h=[];f=0;for(g=a.length;f<g;f++)c=a[f],c.nodeName&&"TR"===c.nodeName.toUpperCase()?h.push(oa(b,c)[0]):h.push(N(b,c));return h},1),c=this.rows(-1);c.pop();h.merge(c,b);return c});p("row()",function(a,b){return db(this.rows(a,b))});p("row().data()",function(a){var b=this.context;if(a===k)return b.length&&this.length?b[0].aoData[this[0]]._aData:
+k;b[0].aoData[this[0]]._aData=a;da(b[0],this[0],"data");return this});p("row().node()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]].nTr||null:null});p("row.add()",function(a){a instanceof h&&a.length&&(a=a[0]);var b=this.iterator("table",function(b){return a.nodeName&&"TR"===a.nodeName.toUpperCase()?oa(b,a)[0]:N(b,a)});return this.row(b[0])});var eb=function(a,b){var c=a.context;if(c.length&&(c=c[0].aoData[b!==k?b:a[0]])&&c._details)c._details.remove(),c._detailsShow=
+k,c._details=k},Wb=function(a,b){var c=a.context;if(c.length&&a.length){var d=c[0].aoData[a[0]];if(d._details){(d._detailsShow=b)?d._details.insertAfter(d.nTr):d._details.detach();var e=c[0],f=new t(e),g=e.aoData;f.off("draw.dt.DT_details column-visibility.dt.DT_details destroy.dt.DT_details");0<D(g,"_details").length&&(f.on("draw.dt.DT_details",function(a,b){e===b&&f.rows({page:"current"}).eq(0).each(function(a){a=g[a];a._detailsShow&&a._details.insertAfter(a.nTr)})}),f.on("column-visibility.dt.DT_details",
+function(a,b){if(e===b)for(var c,d=ba(b),f=0,h=g.length;f<h;f++)c=g[f],c._details&&c._details.children("td[colspan]").attr("colspan",d)}),f.on("destroy.dt.DT_details",function(a,b){if(e===b)for(var c=0,d=g.length;c<d;c++)g[c]._details&&eb(f,c)}))}}};p("row().child()",function(a,b){var c=this.context;if(a===k)return c.length&&this.length?c[0].aoData[this[0]]._details:k;if(!0===a)this.child.show();else if(!1===a)eb(this);else if(c.length&&this.length){var d=c[0],c=c[0].aoData[this[0]],e=[],f=function(a,
+b){if(h.isArray(a)||a instanceof h)for(var c=0,k=a.length;c<k;c++)f(a[c],b);else a.nodeName&&"tr"===a.nodeName.toLowerCase()?e.push(a):(c=h("<tr><td/></tr>").addClass(b),h("td",c).addClass(b).html(a)[0].colSpan=ba(d),e.push(c[0]))};f(a,b);c._details&&c._details.detach();c._details=h(e);c._detailsShow&&c._details.insertAfter(c.nTr)}return this});p(["row().child.show()","row().child().show()"],function(){Wb(this,!0);return this});p(["row().child.hide()","row().child().hide()"],function(){Wb(this,!1);
+return this});p(["row().child.remove()","row().child().remove()"],function(){eb(this);return this});p("row().child.isShown()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var fc=/^([^:]+):(name|visIdx|visible)$/,Xb=function(a,b,c,d,e){for(var c=[],d=0,f=e.length;d<f;d++)c.push(B(a,e[d],b));return c};p("columns()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=cb(b),c=this.iterator("table",function(c){var e=a,f=b,g=c.aoColumns,
+j=D(g,"sName"),i=D(g,"nTh");return bb("column",e,function(a){var b=Qb(a);if(a==="")return X(g.length);if(b!==null)return[b>=0?b:g.length+b];if(typeof a==="function"){var e=Da(c,f);return h.map(g,function(b,f){return a(f,Xb(c,f,0,0,e),i[f])?f:null})}var k=typeof a==="string"?a.match(fc):"";if(k)switch(k[2]){case "visIdx":case "visible":b=parseInt(k[1],10);if(b<0){var m=h.map(g,function(a,b){return a.bVisible?b:null});return[m[m.length+b]]}return[$(c,b)];case "name":return h.map(j,function(a,b){return a===
+k[1]?b:null});default:return[]}if(a.nodeName&&a._DT_CellIndex)return[a._DT_CellIndex.column];b=h(i).filter(a).map(function(){return h.inArray(this,i)}).toArray();if(b.length||!a.nodeName)return b;b=h(a).closest("*[data-dt-column]");return b.length?[b.data("dt-column")]:[]},c,f)},1);c.selector.cols=a;c.selector.opts=b;return c});u("columns().header()","column().header()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTh},1)});u("columns().footer()","column().footer()",
+function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTf},1)});u("columns().data()","column().data()",function(){return this.iterator("column-rows",Xb,1)});u("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData},1)});u("columns().cache()","column().cache()",function(a){return this.iterator("column-rows",function(b,c,d,e,f){return ja(b.aoData,f,"search"===a?"_aFilterData":"_aSortData",c)},1)});u("columns().nodes()",
+"column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,d,e){return ja(a.aoData,e,"anCells",b)},1)});u("columns().visible()","column().visible()",function(a,b){var c=this.iterator("column",function(b,c){if(a===k)return b.aoColumns[c].bVisible;var f=b.aoColumns,g=f[c],j=b.aoData,i,n,l;if(a!==k&&g.bVisible!==a){if(a){var m=h.inArray(!0,D(f,"bVisible"),c+1);i=0;for(n=j.length;i<n;i++)l=j[i].nTr,f=j[i].anCells,l&&l.insertBefore(f[c],f[m]||null)}else h(D(b.aoData,"anCells",c)).detach();
+g.bVisible=a;fa(b,b.aoHeader);fa(b,b.aoFooter);za(b)}});a!==k&&(this.iterator("column",function(c,e){s(c,null,"column-visibility",[c,e,a,b])}),(b===k||b)&&this.columns.adjust());return c});u("columns().indexes()","column().index()",function(a){return this.iterator("column",function(b,c){return"visible"===a?aa(b,c):c},1)});p("columns.adjust()",function(){return this.iterator("table",function(a){Z(a)},1)});p("column.index()",function(a,b){if(0!==this.context.length){var c=this.context[0];if("fromVisible"===
+a||"toData"===a)return $(c,b);if("fromData"===a||"toVisible"===a)return aa(c,b)}});p("column()",function(a,b){return db(this.columns(a,b))});p("cells()",function(a,b,c){h.isPlainObject(a)&&(a.row===k?(c=a,a=null):(c=b,b=null));h.isPlainObject(b)&&(c=b,b=null);if(null===b||b===k)return this.iterator("table",function(b){var d=a,e=cb(c),f=b.aoData,g=Da(b,e),j=Tb(ja(f,g,"anCells")),i=h([].concat.apply([],j)),l,n=b.aoColumns.length,m,p,u,t,s,v;return bb("cell",d,function(a){var c=typeof a==="function";
+if(a===null||a===k||c){m=[];p=0;for(u=g.length;p<u;p++){l=g[p];for(t=0;t<n;t++){s={row:l,column:t};if(c){v=f[l];a(s,B(b,l,t),v.anCells?v.anCells[t]:null)&&m.push(s)}else m.push(s)}}return m}if(h.isPlainObject(a))return[a];c=i.filter(a).map(function(a,b){return{row:b._DT_CellIndex.row,column:b._DT_CellIndex.column}}).toArray();if(c.length||!a.nodeName)return c;v=h(a).closest("*[data-dt-row]");return v.length?[{row:v.data("dt-row"),column:v.data("dt-column")}]:[]},b,e)});var d=this.columns(b,c),e=this.rows(a,
+c),f,g,j,i,n,l=this.iterator("table",function(a,b){f=[];g=0;for(j=e[b].length;g<j;g++){i=0;for(n=d[b].length;i<n;i++)f.push({row:e[b][g],column:d[b][i]})}return f},1);h.extend(l.selector,{cols:b,rows:a,opts:c});return l});u("cells().nodes()","cell().node()",function(){return this.iterator("cell",function(a,b,c){return(a=a.aoData[b])&&a.anCells?a.anCells[c]:k},1)});p("cells().data()",function(){return this.iterator("cell",function(a,b,c){return B(a,b,c)},1)});u("cells().cache()","cell().cache()",function(a){a=
+"search"===a?"_aFilterData":"_aSortData";return this.iterator("cell",function(b,c,d){return b.aoData[c][a][d]},1)});u("cells().render()","cell().render()",function(a){return this.iterator("cell",function(b,c,d){return B(b,c,d,a)},1)});u("cells().indexes()","cell().index()",function(){return this.iterator("cell",function(a,b,c){return{row:b,column:c,columnVisible:aa(a,c)}},1)});u("cells().invalidate()","cell().invalidate()",function(a){return this.iterator("cell",function(b,c,d){da(b,c,a,d)})});p("cell()",
+function(a,b,c){return db(this.cells(a,b,c))});p("cell().data()",function(a){var b=this.context,c=this[0];if(a===k)return b.length&&c.length?B(b[0],c[0].row,c[0].column):k;lb(b[0],c[0].row,c[0].column,a);da(b[0],c[0].row,"data",c[0].column);return this});p("order()",function(a,b){var c=this.context;if(a===k)return 0!==c.length?c[0].aaSorting:k;"number"===typeof a?a=[[a,b]]:a.length&&!h.isArray(a[0])&&(a=Array.prototype.slice.call(arguments));return this.iterator("table",function(b){b.aaSorting=a.slice()})});
+p("order.listener()",function(a,b,c){return this.iterator("table",function(d){Oa(d,a,b,c)})});p("order.fixed()",function(a){if(!a){var b=this.context,b=b.length?b[0].aaSortingFixed:k;return h.isArray(b)?{pre:b}:b}return this.iterator("table",function(b){b.aaSortingFixed=h.extend(!0,{},a)})});p(["columns().order()","column().order()"],function(a){var b=this;return this.iterator("table",function(c,d){var e=[];h.each(b[d],function(b,c){e.push([c,a])});c.aaSorting=e})});p("search()",function(a,b,c,d){var e=
+this.context;return a===k?0!==e.length?e[0].oPreviousSearch.sSearch:k:this.iterator("table",function(e){e.oFeatures.bFilter&&ga(e,h.extend({},e.oPreviousSearch,{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===d?!0:d}),1)})});u("columns().search()","column().search()",function(a,b,c,d){return this.iterator("column",function(e,f){var g=e.aoPreSearchCols;if(a===k)return g[f].sSearch;e.oFeatures.bFilter&&(h.extend(g[f],{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?
+!0:c,bCaseInsensitive:null===d?!0:d}),ga(e,e.oPreviousSearch,1))})});p("state()",function(){return this.context.length?this.context[0].oSavedState:null});p("state.clear()",function(){return this.iterator("table",function(a){a.fnStateSaveCallback.call(a.oInstance,a,{})})});p("state.loaded()",function(){return this.context.length?this.context[0].oLoadedState:null});p("state.save()",function(){return this.iterator("table",function(a){za(a)})});m.versionCheck=m.fnVersionCheck=function(a){for(var b=m.version.split("."),
+a=a.split("."),c,d,e=0,f=a.length;e<f;e++)if(c=parseInt(b[e],10)||0,d=parseInt(a[e],10)||0,c!==d)return c>d;return!0};m.isDataTable=m.fnIsDataTable=function(a){var b=h(a).get(0),c=!1;if(a instanceof m.Api)return!0;h.each(m.settings,function(a,e){var f=e.nScrollHead?h("table",e.nScrollHead)[0]:null,g=e.nScrollFoot?h("table",e.nScrollFoot)[0]:null;if(e.nTable===b||f===b||g===b)c=!0});return c};m.tables=m.fnTables=function(a){var b=!1;h.isPlainObject(a)&&(b=a.api,a=a.visible);var c=h.map(m.settings,
+function(b){if(!a||a&&h(b.nTable).is(":visible"))return b.nTable});return b?new t(c):c};m.camelToHungarian=J;p("$()",function(a,b){var c=this.rows(b).nodes(),c=h(c);return h([].concat(c.filter(a).toArray(),c.find(a).toArray()))});h.each(["on","one","off"],function(a,b){p(b+"()",function(){var a=Array.prototype.slice.call(arguments);a[0]=h.map(a[0].split(/\s/),function(a){return!a.match(/\.dt\b/)?a+".dt":a}).join(" ");var d=h(this.tables().nodes());d[b].apply(d,a);return this})});p("clear()",function(){return this.iterator("table",
+function(a){pa(a)})});p("settings()",function(){return new t(this.context,this.context)});p("init()",function(){var a=this.context;return a.length?a[0].oInit:null});p("data()",function(){return this.iterator("table",function(a){return D(a.aoData,"_aData")}).flatten()});p("destroy()",function(a){a=a||!1;return this.iterator("table",function(b){var c=b.nTableWrapper.parentNode,d=b.oClasses,e=b.nTable,f=b.nTBody,g=b.nTHead,j=b.nTFoot,i=h(e),f=h(f),k=h(b.nTableWrapper),l=h.map(b.aoData,function(a){return a.nTr}),
+p;b.bDestroying=!0;s(b,"aoDestroyCallback","destroy",[b]);a||(new t(b)).columns().visible(!0);k.off(".DT").find(":not(tbody *)").off(".DT");h(E).off(".DT-"+b.sInstance);e!=g.parentNode&&(i.children("thead").detach(),i.append(g));j&&e!=j.parentNode&&(i.children("tfoot").detach(),i.append(j));b.aaSorting=[];b.aaSortingFixed=[];ya(b);h(l).removeClass(b.asStripeClasses.join(" "));h("th, td",g).removeClass(d.sSortable+" "+d.sSortableAsc+" "+d.sSortableDesc+" "+d.sSortableNone);b.bJUI&&(h("th span."+d.sSortIcon+
+", td span."+d.sSortIcon,g).detach(),h("th, td",g).each(function(){var a=h("div."+d.sSortJUIWrapper,this);h(this).append(a.contents());a.detach()}));f.children().detach();f.append(l);g=a?"remove":"detach";i[g]();k[g]();!a&&c&&(c.insertBefore(e,b.nTableReinsertBefore),i.css("width",b.sDestroyWidth).removeClass(d.sTable),(p=b.asDestroyStripes.length)&&f.children().each(function(a){h(this).addClass(b.asDestroyStripes[a%p])}));c=h.inArray(b,m.settings);-1!==c&&m.settings.splice(c,1)})});h.each(["column",
+"row","cell"],function(a,b){p(b+"s().every()",function(a){var d=this.selector.opts,e=this;return this.iterator(b,function(f,g,h,i,m){a.call(e[b](g,"cell"===b?h:d,"cell"===b?d:k),g,h,i,m)})})});p("i18n()",function(a,b,c){var d=this.context[0],a=R(a)(d.oLanguage);a===k&&(a=b);c!==k&&h.isPlainObject(a)&&(a=a[c]!==k?a[c]:a._);return a.replace("%d",c)});m.version="1.10.15";m.settings=[];m.models={};m.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};m.models.oRow={nTr:null,anCells:null,
+_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null,idx:-1};m.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};m.defaults=
+{aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bJQueryUI:!1,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,
+this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+a.sInstance+"_"+location.pathname))}catch(b){}},fnStateLoadParams:null,fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+
+"_"+location.pathname,JSON.stringify(b))}catch(c){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",
+sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},m.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null,rowId:"DT_RowId"};
+Y(m.defaults);m.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};Y(m.defaults.column);m.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,
+bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1,bBounding:!1,barWidth:0},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aIds:{},aoColumns:[],aoHeader:[],aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],
+aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:!0,jqXHR:null,json:k,oAjaxData:k,fnServerData:null,
+aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,bJUI:null,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==y(this)?1*this._iRecordsTotal:this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==y(this)?1*this._iRecordsDisplay:this.aiDisplay.length},fnDisplayEnd:function(){var a=
+this._iDisplayLength,b=this._iDisplayStart,c=b+a,d=this.aiDisplay.length,e=this.oFeatures,f=e.bPaginate;return e.bServerSide?!1===f||-1===a?b+d:Math.min(b+a,this._iRecordsDisplay):!f||c>d||-1===a?d:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{},rowIdFn:null,rowId:null};m.ext=x={buttons:{},classes:{},builder:"-source-",errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},
+header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:m.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:m.version};h.extend(x,{afnFiltering:x.search,aTypes:x.type.detect,ofnSearch:x.type.search,oSort:x.type.order,afnSortData:x.order,aoFeatures:x.feature,oApi:x.internal,oStdClasses:x.classes,oPagination:x.pager});h.extend(m.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",
+sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",
+sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",sJUIHeader:"",sJUIFooter:""});var Ea="",Ea="",G=Ea+"ui-state-default",ka=Ea+"css_right ui-icon ui-icon-",Yb=Ea+"fg-toolbar ui-toolbar ui-widget-header ui-helper-clearfix";h.extend(m.ext.oJUIClasses,
+m.ext.classes,{sPageButton:"fg-button ui-button "+G,sPageButtonActive:"ui-state-disabled",sPageButtonDisabled:"ui-state-disabled",sPaging:"dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",sSortAsc:G+" sorting_asc",sSortDesc:G+" sorting_desc",sSortable:G+" sorting",sSortableAsc:G+" sorting_asc_disabled",sSortableDesc:G+" sorting_desc_disabled",sSortableNone:G+" sorting_disabled",sSortJUIAsc:ka+"triangle-1-n",sSortJUIDesc:ka+"triangle-1-s",sSortJUI:ka+"carat-2-n-s",
+sSortJUIAscAllowed:ka+"carat-1-n",sSortJUIDescAllowed:ka+"carat-1-s",sSortJUIWrapper:"DataTables_sort_wrapper",sSortIcon:"DataTables_sort_icon",sScrollHead:"dataTables_scrollHead "+G,sScrollFoot:"dataTables_scrollFoot "+G,sHeaderTH:G,sFooterTH:G,sJUIHeader:Yb+" ui-corner-tl ui-corner-tr",sJUIFooter:Yb+" ui-corner-bl ui-corner-br"});var Nb=m.ext.pager;h.extend(Nb,{simple:function(){return["previous","next"]},full:function(){return["first","previous","next","last"]},numbers:function(a,b){return[ia(a,
+b)]},simple_numbers:function(a,b){return["previous",ia(a,b),"next"]},full_numbers:function(a,b){return["first","previous",ia(a,b),"next","last"]},first_last_numbers:function(a,b){return["first",ia(a,b),"last"]},_numbers:ia,numbers_length:7});h.extend(!0,m.ext.renderer,{pageButton:{_:function(a,b,c,d,e,f){var g=a.oClasses,j=a.oLanguage.oPaginate,i=a.oLanguage.oAria.paginate||{},m,l,p=0,r=function(b,d){var k,t,u,s,v=function(b){Va(a,b.data.action,true)};k=0;for(t=d.length;k<t;k++){s=d[k];if(h.isArray(s)){u=
+h("<"+(s.DT_el||"div")+"/>").appendTo(b);r(u,s)}else{m=null;l="";switch(s){case "ellipsis":b.append('<span class="ellipsis">…</span>');break;case "first":m=j.sFirst;l=s+(e>0?"":" "+g.sPageButtonDisabled);break;case "previous":m=j.sPrevious;l=s+(e>0?"":" "+g.sPageButtonDisabled);break;case "next":m=j.sNext;l=s+(e<f-1?"":" "+g.sPageButtonDisabled);break;case "last":m=j.sLast;l=s+(e<f-1?"":" "+g.sPageButtonDisabled);break;default:m=s+1;l=e===s?g.sPageButtonActive:""}if(m!==null){u=h("<a>",{"class":g.sPageButton+
+" "+l,"aria-controls":a.sTableId,"aria-label":i[s],"data-dt-idx":p,tabindex:a.iTabIndex,id:c===0&&typeof s==="string"?a.sTableId+"_"+s:null}).html(m).appendTo(b);Ya(u,{action:s},v);p++}}}},t;try{t=h(b).find(H.activeElement).data("dt-idx")}catch(u){}r(h(b).empty(),d);t!==k&&h(b).find("[data-dt-idx="+t+"]").focus()}}});h.extend(m.ext.type.detect,[function(a,b){var c=b.oLanguage.sDecimal;return ab(a,c)?"num"+c:null},function(a){if(a&&!(a instanceof Date)&&!cc.test(a))return null;var b=Date.parse(a);
+return null!==b&&!isNaN(b)||M(a)?"date":null},function(a,b){var c=b.oLanguage.sDecimal;return ab(a,c,!0)?"num-fmt"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Sb(a,c)?"html-num"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Sb(a,c,!0)?"html-num-fmt"+c:null},function(a){return M(a)||"string"===typeof a&&-1!==a.indexOf("<")?"html":null}]);h.extend(m.ext.type.search,{html:function(a){return M(a)?a:"string"===typeof a?a.replace(Pb," ").replace(Ca,""):""},string:function(a){return M(a)?
+a:"string"===typeof a?a.replace(Pb," "):a}});var Ba=function(a,b,c,d){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=Rb(a,b));a.replace&&(c&&(a=a.replace(c,"")),d&&(a=a.replace(d,"")));return 1*a};h.extend(x.type.order,{"date-pre":function(a){return Date.parse(a)||-Infinity},"html-pre":function(a){return M(a)?"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():a+""},"string-pre":function(a){return M(a)?"":"string"===typeof a?a.toLowerCase():!a.toString?"":a.toString()},"string-asc":function(a,b){return a<
+b?-1:a>b?1:0},"string-desc":function(a,b){return a<b?1:a>b?-1:0}});fb("");h.extend(!0,m.ext.renderer,{header:{_:function(a,b,c,d){h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(c.sSortingClass+" "+d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:c.sSortingClass)}})},jqueryui:function(a,b,c,d){h("<div/>").addClass(d.sSortJUIWrapper).append(b.contents()).append(h("<span/>").addClass(d.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);
+h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:c.sSortingClass);b.find("span."+d.sSortIcon).removeClass(d.sSortJUIAsc+" "+d.sSortJUIDesc+" "+d.sSortJUI+" "+d.sSortJUIAscAllowed+" "+d.sSortJUIDescAllowed).addClass(h[e]=="asc"?d.sSortJUIAsc:h[e]=="desc"?d.sSortJUIDesc:c.sSortingClassJUI)}})}}});var Zb=function(a){return"string"===typeof a?a.replace(/</g,"<").replace(/>/g,">").replace(/"/g,
+"""):a};m.render={number:function(a,b,c,d,e){return{display:function(f){if("number"!==typeof f&&"string"!==typeof f)return f;var g=0>f?"-":"",h=parseFloat(f);if(isNaN(h))return Zb(f);h=h.toFixed(c);f=Math.abs(h);h=parseInt(f,10);f=c?b+(f-h).toFixed(c).substring(2):"";return g+(d||"")+h.toString().replace(/\B(?=(\d{3})+(?!\d))/g,a)+f+(e||"")}}},text:function(){return{display:Zb}}};h.extend(m.ext.internal,{_fnExternApiFunc:Ob,_fnBuildAjax:ua,_fnAjaxUpdate:nb,_fnAjaxParameters:wb,_fnAjaxUpdateDraw:xb,
+_fnAjaxDataSrc:va,_fnAddColumn:Ga,_fnColumnOptions:la,_fnAdjustColumnSizing:Z,_fnVisibleToColumnIndex:$,_fnColumnIndexToVisible:aa,_fnVisbleColumns:ba,_fnGetColumns:na,_fnColumnTypes:Ia,_fnApplyColumnDefs:kb,_fnHungarianMap:Y,_fnCamelToHungarian:J,_fnLanguageCompat:Fa,_fnBrowserDetect:ib,_fnAddData:N,_fnAddTr:oa,_fnNodeToDataIndex:function(a,b){return b._DT_RowIndex!==k?b._DT_RowIndex:null},_fnNodeToColumnIndex:function(a,b,c){return h.inArray(c,a.aoData[b].anCells)},_fnGetCellData:B,_fnSetCellData:lb,
+_fnSplitObjNotation:La,_fnGetObjectDataFn:R,_fnSetObjectDataFn:S,_fnGetDataMaster:Ma,_fnClearTable:pa,_fnDeleteIndex:qa,_fnInvalidate:da,_fnGetRowElements:Ka,_fnCreateTr:Ja,_fnBuildHead:mb,_fnDrawHead:fa,_fnDraw:O,_fnReDraw:T,_fnAddOptionsHtml:pb,_fnDetectHeader:ea,_fnGetUniqueThs:ta,_fnFeatureHtmlFilter:rb,_fnFilterComplete:ga,_fnFilterCustom:Ab,_fnFilterColumn:zb,_fnFilter:yb,_fnFilterCreateSearch:Ra,_fnEscapeRegex:Sa,_fnFilterData:Bb,_fnFeatureHtmlInfo:ub,_fnUpdateInfo:Eb,_fnInfoMacros:Fb,_fnInitialise:ha,
+_fnInitComplete:wa,_fnLengthChange:Ta,_fnFeatureHtmlLength:qb,_fnFeatureHtmlPaginate:vb,_fnPageChange:Va,_fnFeatureHtmlProcessing:sb,_fnProcessingDisplay:C,_fnFeatureHtmlTable:tb,_fnScrollDraw:ma,_fnApplyToChildren:I,_fnCalculateColumnWidths:Ha,_fnThrottle:Qa,_fnConvertToWidth:Gb,_fnGetWidestNode:Hb,_fnGetMaxLenString:Ib,_fnStringToCss:v,_fnSortFlatten:W,_fnSort:ob,_fnSortAria:Kb,_fnSortListener:Xa,_fnSortAttachListener:Oa,_fnSortingClasses:ya,_fnSortData:Jb,_fnSaveState:za,_fnLoadState:Lb,_fnSettingsFromNode:Aa,
+_fnLog:K,_fnMap:F,_fnBindAction:Ya,_fnCallbackReg:z,_fnCallbackFire:s,_fnLengthOverflow:Ua,_fnRenderer:Pa,_fnDataSource:y,_fnRowAttributes:Na,_fnCalculateEnd:function(){}});h.fn.dataTable=m;m.$=h;h.fn.dataTableSettings=m.settings;h.fn.dataTableExt=m.ext;h.fn.DataTable=function(a){return h(this).dataTable(a).api()};h.each(m,function(a,b){h.fn.DataTable[a]=b});return h.fn.dataTable});
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/livy-mini-logo.png
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/livy-mini-logo.png b/server/src/main/resources/org/apache/livy/server/ui/static/livy-mini-logo.png
new file mode 100644
index 0000000..49606b4
Binary files /dev/null and b/server/src/main/resources/org/apache/livy/server/ui/static/livy-mini-logo.png differ
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/livy-ui.css
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/livy-ui.css b/server/src/main/resources/org/apache/livy/server/ui/static/livy-ui.css
new file mode 100644
index 0000000..aadb256
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/livy-ui.css
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+body {
+ padding-top: 20px;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/org/apache/livy/server/ui/static/sessions-table.html
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/sessions-table.html b/server/src/main/resources/org/apache/livy/server/ui/static/sessions-table.html
new file mode 100644
index 0000000..a50d328
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/sessions-table.html
@@ -0,0 +1,59 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h4 id="interactive-sessions-header" class="sessions-template">Interactive Sessions</h4>
+
+<table id="interactive-sessions-table"
+ class="table table-striped sessions-table sessions-template">
+ <thead class="sessions-table-head">
+ <tr>
+ <th>Session Id</th>
+ <th>
+ <span data-toggle="tooltip"
+ title="Spark Application Id for this session.
+ If available, links to Spark Application Web UI">
+ Application Id
+ </span>
+ </th>
+ <th>
+ <span data-toggle="tooltip" title="Remote user who submitted this session">
+ Owner
+ </span>
+ </th>
+ <th>
+ <span data-toggle="tooltip" title="User to impersonate when running">
+ Proxy User
+ </span>
+ </th>
+ <th>
+ <span data-toggle="tooltip"
+ title="Session kind (spark, pyspark, pyspark3, or sparkr)">
+ Session Kind
+ </span>
+ </th>
+ <th>
+ <span data-toggle="tooltip"
+ title="Session State (not_started, starting, idle, busy,
+ shutting_down, error, dead, success)">
+ State
+ </span>
+ </th>
+ </tr>
+ </thead>
+ <tbody class="sessions-table-body">
+ </tbody>
+</table>
\ No newline at end of file
[16/33] incubator-livy git commit: LIVY-375. Change Livy code package
name to org.apache.livy
Posted by js...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery-3.2.1.min.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery-3.2.1.min.js b/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery-3.2.1.min.js
deleted file mode 100644
index 644d35e..0000000
--- a/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery-3.2.1.min.js
+++ /dev/null
@@ -1,4 +0,0 @@
-/*! jQuery v3.2.1 | (c) JS Foundation and other contributors | jquery.org/license */
-!function(a,b){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){"use strict";var c=[],d=a.document,e=Object.getPrototypeOf,f=c.slice,g=c.concat,h=c.push,i=c.indexOf,j={},k=j.toString,l=j.hasOwnProperty,m=l.toString,n=m.call(Object),o={};function p(a,b){b=b||d;var c=b.createElement("script");c.text=a,b.head.appendChild(c).parentNode.removeChild(c)}var q="3.2.1",r=function(a,b){return new r.fn.init(a,b)},s=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,t=/^-ms-/,u=/-([a-z])/g,v=function(a,b){return b.toUpperCase()};r.fn=r.prototype={jquery:q,constructor:r,length:0,toArray:function(){return f.call(this)},get:function(a){return null==a?f.call(this):a<0?this[a+this.length]:this[a]},pushStack:function(a){var b=r.merge(this.constructor(),a);return b.prevObject=this,b},each:function(a){retu
rn r.each(this,a)},map:function(a){return this.pushStack(r.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(f.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(a<0?b:0);return this.pushStack(c>=0&&c<b?[this[c]]:[])},end:function(){return this.prevObject||this.constructor()},push:h,sort:c.sort,splice:c.splice},r.extend=r.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||r.isFunction(g)||(g={}),h===i&&(g=this,h--);h<i;h++)if(null!=(a=arguments[h]))for(b in a)c=g[b],d=a[b],g!==d&&(j&&d&&(r.isPlainObject(d)||(e=Array.isArray(d)))?(e?(e=!1,f=c&&Array.isArray(c)?c:[]):f=c&&r.isPlainObject(c)?c:{},g[b]=r.extend(j,f,d)):void 0!==d&&(g[b]=d));return g},r.extend({expando:"jQuery"+(q+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},no
op:function(){},isFunction:function(a){return"function"===r.type(a)},isWindow:function(a){return null!=a&&a===a.window},isNumeric:function(a){var b=r.type(a);return("number"===b||"string"===b)&&!isNaN(a-parseFloat(a))},isPlainObject:function(a){var b,c;return!(!a||"[object Object]"!==k.call(a))&&(!(b=e(a))||(c=l.call(b,"constructor")&&b.constructor,"function"==typeof c&&m.call(c)===n))},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?j[k.call(a)]||"object":typeof a},globalEval:function(a){p(a)},camelCase:function(a){return a.replace(t,"ms-").replace(u,v)},each:function(a,b){var c,d=0;if(w(a)){for(c=a.length;d<c;d++)if(b.call(a[d],d,a[d])===!1)break}else for(d in a)if(b.call(a[d],d,a[d])===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(s,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(w(Object(a))?r.merge(c,"string"==typeof a?[a]:a):h.call(c,a)),c},inArray:funct
ion(a,b,c){return null==b?-1:i.call(b,a,c)},merge:function(a,b){for(var c=+b.length,d=0,e=a.length;d<c;d++)a[e++]=b[d];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;f<g;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,e,f=0,h=[];if(w(a))for(d=a.length;f<d;f++)e=b(a[f],f,c),null!=e&&h.push(e);else for(f in a)e=b(a[f],f,c),null!=e&&h.push(e);return g.apply([],h)},guid:1,proxy:function(a,b){var c,d,e;if("string"==typeof b&&(c=a[b],b=a,a=c),r.isFunction(a))return d=f.call(arguments,2),e=function(){return a.apply(b||this,d.concat(f.call(arguments)))},e.guid=a.guid=a.guid||r.guid++,e},now:Date.now,support:o}),"function"==typeof Symbol&&(r.fn[Symbol.iterator]=c[Symbol.iterator]),r.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(a,b){j["[object "+b+"]"]=b.toLowerCase()});function w(a){var b=!!a&&"length"in a&&a.length,c=r.type(a);return"function"!==c&&!r.isWindow(a)&&("array"===c||0===b||"
number"==typeof b&&b>0&&b-1 in a)}var x=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=ha(),z=ha(),A=ha(),B=function(a,b){return a===b&&(l=!0),0},C={}.hasOwnProperty,D=[],E=D.pop,F=D.push,G=D.push,H=D.slice,I=function(a,b){for(var c=0,d=a.length;c<d;c++)if(a[c]===b)return c;return-1},J="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",K="[\\x20\\t\\r\\n\\f]",L="(?:\\\\.|[\\w-]|[^\0-\\xa0])+",M="\\["+K+"*("+L+")(?:"+K+"*([*^$|!~]?=)"+K+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+L+"))|)"+K+"*\\]",N=":("+L+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+M+")*)|.*)\\)|)",O=new RegExp(K+"+","g"),P=new RegExp("^"+K+"+|((?:^|[^\\\\])(?:\\\\.)*)"+K+"+$","g"),Q=new RegExp("^"+K+"*,"+K+"*"),R=new RegExp("^"+K+"*([>+~]|"+K+")"+K+"*"),S=new RegExp("="+K+"*([^\\]'\"]*?)"+K+"*\\]","g"),T=new RegExp(N),U=new RegExp("^
"+L+"$"),V={ID:new RegExp("^#("+L+")"),CLASS:new RegExp("^\\.("+L+")"),TAG:new RegExp("^("+L+"|[*])"),ATTR:new RegExp("^"+M),PSEUDO:new RegExp("^"+N),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+K+"*(even|odd|(([+-]|)(\\d*)n|)"+K+"*(?:([+-]|)"+K+"*(\\d+)|))"+K+"*\\)|)","i"),bool:new RegExp("^(?:"+J+")$","i"),needsContext:new RegExp("^"+K+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+K+"*((?:-\\d)?\\d*)"+K+"*\\)|)(?=[^-]|$)","i")},W=/^(?:input|select|textarea|button)$/i,X=/^h\d$/i,Y=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,$=/[+~]/,_=new RegExp("\\\\([\\da-f]{1,6}"+K+"?|("+K+")|.)","ig"),aa=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:d<0?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},ba=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ca=function(a,b){return b?"\0"===a?"\ufffd":a.slice(0,-1)+"\\"+a.charCodeAt(a.length-1).toString(16)+" ":"\\"+a},da=function(){m()},ea=ta(function(a){retur
n a.disabled===!0&&("form"in a||"label"in a)},{dir:"parentNode",next:"legend"});try{G.apply(D=H.call(v.childNodes),v.childNodes),D[v.childNodes.length].nodeType}catch(fa){G={apply:D.length?function(a,b){F.apply(a,H.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function ga(a,b,d,e){var f,h,j,k,l,o,r,s=b&&b.ownerDocument,w=b?b.nodeType:9;if(d=d||[],"string"!=typeof a||!a||1!==w&&9!==w&&11!==w)return d;if(!e&&((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,p)){if(11!==w&&(l=Z.exec(a)))if(f=l[1]){if(9===w){if(!(j=b.getElementById(f)))return d;if(j.id===f)return d.push(j),d}else if(s&&(j=s.getElementById(f))&&t(b,j)&&j.id===f)return d.push(j),d}else{if(l[2])return G.apply(d,b.getElementsByTagName(a)),d;if((f=l[3])&&c.getElementsByClassName&&b.getElementsByClassName)return G.apply(d,b.getElementsByClassName(f)),d}if(c.qsa&&!A[a+" "]&&(!q||!q.test(a))){if(1!==w)s=b,r=a;else if("object"!==b.nodeName.toLowerCase()){(k=b.getAttribute("id"))?k=k.replace(ba,ca):b.set
Attribute("id",k=u),o=g(a),h=o.length;while(h--)o[h]="#"+k+" "+sa(o[h]);r=o.join(","),s=$.test(a)&&qa(b.parentNode)||b}if(r)try{return G.apply(d,s.querySelectorAll(r)),d}catch(x){}finally{k===u&&b.removeAttribute("id")}}}return i(a.replace(P,"$1"),b,d,e)}function ha(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ia(a){return a[u]=!0,a}function ja(a){var b=n.createElement("fieldset");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function ka(a,b){var c=a.split("|"),e=c.length;while(e--)d.attrHandle[c[e]]=b}function la(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&a.sourceIndex-b.sourceIndex;if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function ma(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function na(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}f
unction oa(a){return function(b){return"form"in b?b.parentNode&&b.disabled===!1?"label"in b?"label"in b.parentNode?b.parentNode.disabled===a:b.disabled===a:b.isDisabled===a||b.isDisabled!==!a&&ea(b)===a:b.disabled===a:"label"in b&&b.disabled===a}}function pa(a){return ia(function(b){return b=+b,ia(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function qa(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=ga.support={},f=ga.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return!!b&&"HTML"!==b.nodeName},m=ga.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=n.documentElement,p=!f(n),v!==n&&(e=n.defaultView)&&e.top!==e&&(e.addEventListener?e.addEventListener("unload",da,!1):e.attachEvent&&e.attachEvent("onunload",da)),c.attributes=ja(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=ja(function(a){return a.append
Child(n.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=Y.test(n.getElementsByClassName),c.getById=ja(function(a){return o.appendChild(a).id=u,!n.getElementsByName||!n.getElementsByName(u).length}),c.getById?(d.filter.ID=function(a){var b=a.replace(_,aa);return function(a){return a.getAttribute("id")===b}},d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c?[c]:[]}}):(d.filter.ID=function(a){var b=a.replace(_,aa);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}},d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c,d,e,f=b.getElementById(a);if(f){if(c=f.getAttributeNode("id"),c&&c.value===a)return[f];e=b.getElementsByName(a),d=0;while(f=e[d++])if(c=f.getAttributeNode("id"),c&&c.value===a)return[f]}return[]}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElement
sByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){if("undefined"!=typeof b.getElementsByClassName&&p)return b.getElementsByClassName(a)},r=[],q=[],(c.qsa=Y.test(n.querySelectorAll))&&(ja(function(a){o.appendChild(a).innerHTML="<a id='"+u+"'></a><select id='"+u+"-\r\\' msallowcapture=''><option selected=''></option></select>",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+K+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+K+"*(?:value|"+J+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),ja(function(a){a.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var b=n.createElement("input");b.setAttri
bute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+K+"*[*^$|!~]?="),2!==a.querySelectorAll(":enabled").length&&q.push(":enabled",":disabled"),o.appendChild(a).disabled=!0,2!==a.querySelectorAll(":disabled").length&&q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=Y.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&ja(function(a){c.disconnectedMatch=s.call(a,"*"),s.call(a,"[s!='']:x"),r.push("!=",N)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=Y.test(o.compareDocumentPosition),t=b||Y.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===
b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===n||a.ownerDocument===v&&t(v,a)?-1:b===n||b.ownerDocument===v&&t(v,b)?1:k?I(k,a)-I(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,g=[a],h=[b];if(!e||!f)return a===n?-1:b===n?1:e?-1:f?1:k?I(k,a)-I(k,b):0;if(e===f)return la(a,b);c=a;while(c=c.parentNode)g.unshift(c);c=b;while(c=c.parentNode)h.unshift(c);while(g[d]===h[d])d++;return d?la(g[d],h[d]):g[d]===v?-1:h[d]===v?1:0},n):n},ga.matches=function(a,b){return ga(a,null,null,b)},ga.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(S,"='$1']"),c.matchesSelector&&p&&!A[b+" "]&&(!r||!r.test(b))&&(!q||!q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return ga(b,n,null,[a]).length>
0},ga.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},ga.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&C.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},ga.escape=function(a){return(a+"").replace(ba,ca)},ga.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},ga.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=ga.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=ga.selectors={cacheLength:50,createPseudo:ia,match:V,attrHandle:{},find:{},
relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(_,aa),a[3]=(a[3]||a[4]||a[5]||"").replace(_,aa),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||ga.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&ga.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return V.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&T.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(_,aa).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+K+")"+a+"("+K+"|$)"))&&y(a,function(a){return b.test("string"==typ
eof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=ga.attr(d,a);return null==e?"!="===b:!b||(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(O," ")+" ").indexOf(c)>-1:"|="===b&&(e===c||e.slice(0,c.length+1)===c+"-"))}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h,t=!1;if(q){if(f){while(p){m=b;while(m=m[p])if(h?m.nodeName.toLowerCase()===r:1===m.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){m=q,l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),j=k[a]||[],n=j[0]===w&&j[1],t=n&&j[2],m=n&&q.childNodes[n];while(m=+
+n&&m&&m[p]||(t=n=0)||o.pop())if(1===m.nodeType&&++t&&m===b){k[a]=[w,n,t];break}}else if(s&&(m=b,l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),j=k[a]||[],n=j[0]===w&&j[1],t=n),t===!1)while(m=++n&&m&&m[p]||(t=n=0)||o.pop())if((h?m.nodeName.toLowerCase()===r:1===m.nodeType)&&++t&&(s&&(l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),k[a]=[w,t]),m===b))break;return t-=e,t===d||t%d===0&&t/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||ga.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ia(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=I(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ia(function(a){var b=[],c=[],d=h(a.replace(P,"$1"));return d[u]?ia(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ia(function(a){return function(b
){return ga(a,b).length>0}}),contains:ia(function(a){return a=a.replace(_,aa),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ia(function(a){return U.test(a||"")||ga.error("unsupported lang: "+a),a=a.replace(_,aa).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:oa(!1),disabled:oa(!0),checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:funct
ion(a){return!d.pseudos.empty(a)},header:function(a){return X.test(a.nodeName)},input:function(a){return W.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:pa(function(){return[0]}),last:pa(function(a,b){return[b-1]}),eq:pa(function(a,b,c){return[c<0?c+b:c]}),even:pa(function(a,b){for(var c=0;c<b;c+=2)a.push(c);return a}),odd:pa(function(a,b){for(var c=1;c<b;c+=2)a.push(c);return a}),lt:pa(function(a,b,c){for(var d=c<0?c+b:c;--d>=0;)a.push(d);return a}),gt:pa(function(a,b,c){for(var d=c<0?c+b:c;++d<b;)a.push(d);return a})}},d.pseudos.nth=d.pseudos.eq;for(b in{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})d.pseudos[b]=ma(b);for(b in{submit:!0,reset:!0})d.pseudos[b]=na(b);function ra(){}ra.prototype=d.filters=d.pseudos,d.setFilters=new ra,g=ga.tokenize=function(a
,b){var c,e,f,g,h,i,j,k=z[a+" "];if(k)return b?0:k.slice(0);h=a,i=[],j=d.preFilter;while(h){c&&!(e=Q.exec(h))||(e&&(h=h.slice(e[0].length)||h),i.push(f=[])),c=!1,(e=R.exec(h))&&(c=e.shift(),f.push({value:c,type:e[0].replace(P," ")}),h=h.slice(c.length));for(g in d.filter)!(e=V[g].exec(h))||j[g]&&!(e=j[g](e))||(c=e.shift(),f.push({value:c,type:g,matches:e}),h=h.slice(c.length));if(!c)break}return b?h.length:h?ga.error(a):z(a,i).slice(0)};function sa(a){for(var b=0,c=a.length,d="";b<c;b++)d+=a[b].value;return d}function ta(a,b,c){var d=b.dir,e=b.next,f=e||d,g=c&&"parentNode"===f,h=x++;return b.first?function(b,c,e){while(b=b[d])if(1===b.nodeType||g)return a(b,c,e);return!1}:function(b,c,i){var j,k,l,m=[w,h];if(i){while(b=b[d])if((1===b.nodeType||g)&&a(b,c,i))return!0}else while(b=b[d])if(1===b.nodeType||g)if(l=b[u]||(b[u]={}),k=l[b.uniqueID]||(l[b.uniqueID]={}),e&&e===b.nodeName.toLowerCase())b=b[d]||b;else{if((j=k[f])&&j[0]===w&&j[1]===h)return m[2]=j[2];if(k[f]=m,m[2]=a(b,c,i))retur
n!0}return!1}}function ua(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function va(a,b,c){for(var d=0,e=b.length;d<e;d++)ga(a,b[d],c);return c}function wa(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;h<i;h++)(f=a[h])&&(c&&!c(f,d,e)||(g.push(f),j&&b.push(h)));return g}function xa(a,b,c,d,e,f){return d&&!d[u]&&(d=xa(d)),e&&!e[u]&&(e=xa(e,f)),ia(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||va(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:wa(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=wa(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?I(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=wa(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):G.apply(g,r)})}function ya(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=ta(function(a){return a===b},
h,!0),l=ta(function(a){return I(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];i<f;i++)if(c=d.relative[a[i].type])m=[ta(ua(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;e<f;e++)if(d.relative[a[e].type])break;return xa(i>1&&ua(m),i>1&&sa(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(P,"$1"),c,i<e&&ya(a.slice(i,e)),e<f&&ya(a=a.slice(e)),e<f&&sa(a))}m.push(c)}return ua(m)}function za(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,o,q,r=0,s="0",t=f&&[],u=[],v=j,x=f||e&&d.find.TAG("*",k),y=w+=null==v?1:Math.random()||.1,z=x.length;for(k&&(j=g===n||g||k);s!==z&&null!=(l=x[s]);s++){if(e&&l){o=0,g||l.ownerDocument===n||(m(l),h=!p);while(q=a[o++])if(q(l,g||n,h)){i.push(l);break}k&&(w=y)}c&&((l=!q&&l)&&r--,f&&t.push(l))}if(r+=s,c&&s!==r){o=0;while(q=b[o++])q(t,u,g,h);if(f){if(r>0)while(s--)t[s]||u[s]||(u[s]=E.call(i));u=wa(u)}G.apply(i,u),k&&!f&&u.length>0&&r+b.leng
th>1&&ga.uniqueSort(i)}return k&&(w=y,j=v),t};return c?ia(f):f}return h=ga.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=ya(b[c]),f[u]?d.push(f):e.push(f);f=A(a,za(e,d)),f.selector=a}return f},i=ga.select=function(a,b,c,e){var f,i,j,k,l,m="function"==typeof a&&a,n=!e&&g(a=m.selector||a);if(c=c||[],1===n.length){if(i=n[0]=n[0].slice(0),i.length>2&&"ID"===(j=i[0]).type&&9===b.nodeType&&p&&d.relative[i[1].type]){if(b=(d.find.ID(j.matches[0].replace(_,aa),b)||[])[0],!b)return c;m&&(b=b.parentNode),a=a.slice(i.shift().value.length)}f=V.needsContext.test(a)?0:i.length;while(f--){if(j=i[f],d.relative[k=j.type])break;if((l=d.find[k])&&(e=l(j.matches[0].replace(_,aa),$.test(i[0].type)&&qa(b.parentNode)||b))){if(i.splice(f,1),a=e.length&&sa(i),!a)return G.apply(c,e),c;break}}}return(m||h(a,n))(e,b,!p,c,!b||$.test(a)&&qa(b.parentNode)||b),c},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=ja(function(a){return
1&a.compareDocumentPosition(n.createElement("fieldset"))}),ja(function(a){return a.innerHTML="<a href='#'></a>","#"===a.firstChild.getAttribute("href")})||ka("type|href|height|width",function(a,b,c){if(!c)return a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&ja(function(a){return a.innerHTML="<input/>",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||ka("value",function(a,b,c){if(!c&&"input"===a.nodeName.toLowerCase())return a.defaultValue}),ja(function(a){return null==a.getAttribute("disabled")})||ka(J,function(a,b,c){var d;if(!c)return a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),ga}(a);r.find=x,r.expr=x.selectors,r.expr[":"]=r.expr.pseudos,r.uniqueSort=r.unique=x.uniqueSort,r.text=x.getText,r.isXMLDoc=x.isXML,r.contains=x.contains,r.escapeSelector=x.escape;var y=function(a,b,c){var d=[],e=void 0!==c;while((a=a[b])&&9!==a.nodeType)if(1===a.nodeType){if(e&&r(a).is(c))break;d.push(a)}return d},z=func
tion(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c},A=r.expr.match.needsContext;function B(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()}var C=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i,D=/^.[^:#\[\.,]*$/;function E(a,b,c){return r.isFunction(b)?r.grep(a,function(a,d){return!!b.call(a,d,a)!==c}):b.nodeType?r.grep(a,function(a){return a===b!==c}):"string"!=typeof b?r.grep(a,function(a){return i.call(b,a)>-1!==c}):D.test(b)?r.filter(b,a,c):(b=r.filter(b,a),r.grep(a,function(a){return i.call(b,a)>-1!==c&&1===a.nodeType}))}r.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?r.find.matchesSelector(d,a)?[d]:[]:r.find.matches(a,r.grep(b,function(a){return 1===a.nodeType}))},r.fn.extend({find:function(a){var b,c,d=this.length,e=this;if("string"!=typeof a)return this.pushStack(r(a).filter(function(){for(b=0;b<d;b++)if(r.contains(e[b],this))return!0}));for(c=this.pushStack([]),
b=0;b<d;b++)r.find(a,e[b],c);return d>1?r.uniqueSort(c):c},filter:function(a){return this.pushStack(E(this,a||[],!1))},not:function(a){return this.pushStack(E(this,a||[],!0))},is:function(a){return!!E(this,"string"==typeof a&&A.test(a)?r(a):a||[],!1).length}});var F,G=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/,H=r.fn.init=function(a,b,c){var e,f;if(!a)return this;if(c=c||F,"string"==typeof a){if(e="<"===a[0]&&">"===a[a.length-1]&&a.length>=3?[null,a,null]:G.exec(a),!e||!e[1]&&b)return!b||b.jquery?(b||c).find(a):this.constructor(b).find(a);if(e[1]){if(b=b instanceof r?b[0]:b,r.merge(this,r.parseHTML(e[1],b&&b.nodeType?b.ownerDocument||b:d,!0)),C.test(e[1])&&r.isPlainObject(b))for(e in b)r.isFunction(this[e])?this[e](b[e]):this.attr(e,b[e]);return this}return f=d.getElementById(e[2]),f&&(this[0]=f,this.length=1),this}return a.nodeType?(this[0]=a,this.length=1,this):r.isFunction(a)?void 0!==c.ready?c.ready(a):a(r):r.makeArray(a,this)};H.prototype=r.fn,F=r(d);var I=/^(?:parents|prev(?:Until|
All))/,J={children:!0,contents:!0,next:!0,prev:!0};r.fn.extend({has:function(a){var b=r(a,this),c=b.length;return this.filter(function(){for(var a=0;a<c;a++)if(r.contains(this,b[a]))return!0})},closest:function(a,b){var c,d=0,e=this.length,f=[],g="string"!=typeof a&&r(a);if(!A.test(a))for(;d<e;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&r.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?r.uniqueSort(f):f)},index:function(a){return a?"string"==typeof a?i.call(r(a),this[0]):i.call(this,a.jquery?a[0]:a):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(r.uniqueSort(r.merge(this.get(),r(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function K(a,b){while((a=a[b])&&1!==a.nodeType);return a}r.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return y(a,"parentNode
")},parentsUntil:function(a,b,c){return y(a,"parentNode",c)},next:function(a){return K(a,"nextSibling")},prev:function(a){return K(a,"previousSibling")},nextAll:function(a){return y(a,"nextSibling")},prevAll:function(a){return y(a,"previousSibling")},nextUntil:function(a,b,c){return y(a,"nextSibling",c)},prevUntil:function(a,b,c){return y(a,"previousSibling",c)},siblings:function(a){return z((a.parentNode||{}).firstChild,a)},children:function(a){return z(a.firstChild)},contents:function(a){return B(a,"iframe")?a.contentDocument:(B(a,"template")&&(a=a.content||a),r.merge([],a.childNodes))}},function(a,b){r.fn[a]=function(c,d){var e=r.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=r.filter(d,e)),this.length>1&&(J[a]||r.uniqueSort(e),I.test(a)&&e.reverse()),this.pushStack(e)}});var L=/[^\x20\t\r\n\f]+/g;function M(a){var b={};return r.each(a.match(L)||[],function(a,c){b[c]=!0}),b}r.Callbacks=function(a){a="string"==typeof a?M(a):r.extend({},a);var b,c,d,e,f=
[],g=[],h=-1,i=function(){for(e=e||a.once,d=b=!0;g.length;h=-1){c=g.shift();while(++h<f.length)f[h].apply(c[0],c[1])===!1&&a.stopOnFalse&&(h=f.length,c=!1)}a.memory||(c=!1),b=!1,e&&(f=c?[]:"")},j={add:function(){return f&&(c&&!b&&(h=f.length-1,g.push(c)),function d(b){r.each(b,function(b,c){r.isFunction(c)?a.unique&&j.has(c)||f.push(c):c&&c.length&&"string"!==r.type(c)&&d(c)})}(arguments),c&&!b&&i()),this},remove:function(){return r.each(arguments,function(a,b){var c;while((c=r.inArray(b,f,c))>-1)f.splice(c,1),c<=h&&h--}),this},has:function(a){return a?r.inArray(a,f)>-1:f.length>0},empty:function(){return f&&(f=[]),this},disable:function(){return e=g=[],f=c="",this},disabled:function(){return!f},lock:function(){return e=g=[],c||b||(f=c=""),this},locked:function(){return!!e},fireWith:function(a,c){return e||(c=c||[],c=[a,c.slice?c.slice():c],g.push(c),b||i()),this},fire:function(){return j.fireWith(this,arguments),this},fired:function(){return!!d}};return j};function N(a){return a}fu
nction O(a){throw a}function P(a,b,c,d){var e;try{a&&r.isFunction(e=a.promise)?e.call(a).done(b).fail(c):a&&r.isFunction(e=a.then)?e.call(a,b,c):b.apply(void 0,[a].slice(d))}catch(a){c.apply(void 0,[a])}}r.extend({Deferred:function(b){var c=[["notify","progress",r.Callbacks("memory"),r.Callbacks("memory"),2],["resolve","done",r.Callbacks("once memory"),r.Callbacks("once memory"),0,"resolved"],["reject","fail",r.Callbacks("once memory"),r.Callbacks("once memory"),1,"rejected"]],d="pending",e={state:function(){return d},always:function(){return f.done(arguments).fail(arguments),this},"catch":function(a){return e.then(null,a)},pipe:function(){var a=arguments;return r.Deferred(function(b){r.each(c,function(c,d){var e=r.isFunction(a[d[4]])&&a[d[4]];f[d[1]](function(){var a=e&&e.apply(this,arguments);a&&r.isFunction(a.promise)?a.promise().progress(b.notify).done(b.resolve).fail(b.reject):b[d[0]+"With"](this,e?[a]:arguments)})}),a=null}).promise()},then:function(b,d,e){var f=0;function g(b
,c,d,e){return function(){var h=this,i=arguments,j=function(){var a,j;if(!(b<f)){if(a=d.apply(h,i),a===c.promise())throw new TypeError("Thenable self-resolution");j=a&&("object"==typeof a||"function"==typeof a)&&a.then,r.isFunction(j)?e?j.call(a,g(f,c,N,e),g(f,c,O,e)):(f++,j.call(a,g(f,c,N,e),g(f,c,O,e),g(f,c,N,c.notifyWith))):(d!==N&&(h=void 0,i=[a]),(e||c.resolveWith)(h,i))}},k=e?j:function(){try{j()}catch(a){r.Deferred.exceptionHook&&r.Deferred.exceptionHook(a,k.stackTrace),b+1>=f&&(d!==O&&(h=void 0,i=[a]),c.rejectWith(h,i))}};b?k():(r.Deferred.getStackHook&&(k.stackTrace=r.Deferred.getStackHook()),a.setTimeout(k))}}return r.Deferred(function(a){c[0][3].add(g(0,a,r.isFunction(e)?e:N,a.notifyWith)),c[1][3].add(g(0,a,r.isFunction(b)?b:N)),c[2][3].add(g(0,a,r.isFunction(d)?d:O))}).promise()},promise:function(a){return null!=a?r.extend(a,e):e}},f={};return r.each(c,function(a,b){var g=b[2],h=b[5];e[b[1]]=g.add,h&&g.add(function(){d=h},c[3-a][2].disable,c[0][2].lock),g.add(b[3].fire),
f[b[0]]=function(){return f[b[0]+"With"](this===f?void 0:this,arguments),this},f[b[0]+"With"]=g.fireWith}),e.promise(f),b&&b.call(f,f),f},when:function(a){var b=arguments.length,c=b,d=Array(c),e=f.call(arguments),g=r.Deferred(),h=function(a){return function(c){d[a]=this,e[a]=arguments.length>1?f.call(arguments):c,--b||g.resolveWith(d,e)}};if(b<=1&&(P(a,g.done(h(c)).resolve,g.reject,!b),"pending"===g.state()||r.isFunction(e[c]&&e[c].then)))return g.then();while(c--)P(e[c],h(c),g.reject);return g.promise()}});var Q=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;r.Deferred.exceptionHook=function(b,c){a.console&&a.console.warn&&b&&Q.test(b.name)&&a.console.warn("jQuery.Deferred exception: "+b.message,b.stack,c)},r.readyException=function(b){a.setTimeout(function(){throw b})};var R=r.Deferred();r.fn.ready=function(a){return R.then(a)["catch"](function(a){r.readyException(a)}),this},r.extend({isReady:!1,readyWait:1,ready:function(a){(a===!0?--r.readyWait:r.isReady)||(r.isReady=!
0,a!==!0&&--r.readyWait>0||R.resolveWith(d,[r]))}}),r.ready.then=R.then;function S(){d.removeEventListener("DOMContentLoaded",S),
-a.removeEventListener("load",S),r.ready()}"complete"===d.readyState||"loading"!==d.readyState&&!d.documentElement.doScroll?a.setTimeout(r.ready):(d.addEventListener("DOMContentLoaded",S),a.addEventListener("load",S));var T=function(a,b,c,d,e,f,g){var h=0,i=a.length,j=null==c;if("object"===r.type(c)){e=!0;for(h in c)T(a,b,h,c[h],!0,f,g)}else if(void 0!==d&&(e=!0,r.isFunction(d)||(g=!0),j&&(g?(b.call(a,d),b=null):(j=b,b=function(a,b,c){return j.call(r(a),c)})),b))for(;h<i;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},U=function(a){return 1===a.nodeType||9===a.nodeType||!+a.nodeType};function V(){this.expando=r.expando+V.uid++}V.uid=1,V.prototype={cache:function(a){var b=a[this.expando];return b||(b={},U(a)&&(a.nodeType?a[this.expando]=b:Object.defineProperty(a,this.expando,{value:b,configurable:!0}))),b},set:function(a,b,c){var d,e=this.cache(a);if("string"==typeof b)e[r.camelCase(b)]=c;else for(d in b)e[r.camelCase(d)]=b[d];return e},get:function(a,
b){return void 0===b?this.cache(a):a[this.expando]&&a[this.expando][r.camelCase(b)]},access:function(a,b,c){return void 0===b||b&&"string"==typeof b&&void 0===c?this.get(a,b):(this.set(a,b,c),void 0!==c?c:b)},remove:function(a,b){var c,d=a[this.expando];if(void 0!==d){if(void 0!==b){Array.isArray(b)?b=b.map(r.camelCase):(b=r.camelCase(b),b=b in d?[b]:b.match(L)||[]),c=b.length;while(c--)delete d[b[c]]}(void 0===b||r.isEmptyObject(d))&&(a.nodeType?a[this.expando]=void 0:delete a[this.expando])}},hasData:function(a){var b=a[this.expando];return void 0!==b&&!r.isEmptyObject(b)}};var W=new V,X=new V,Y=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,Z=/[A-Z]/g;function $(a){return"true"===a||"false"!==a&&("null"===a?null:a===+a+""?+a:Y.test(a)?JSON.parse(a):a)}function _(a,b,c){var d;if(void 0===c&&1===a.nodeType)if(d="data-"+b.replace(Z,"-$&").toLowerCase(),c=a.getAttribute(d),"string"==typeof c){try{c=$(c)}catch(e){}X.set(a,b,c)}else c=void 0;return c}r.extend({hasData:function(a){return X.hasData(a)|
|W.hasData(a)},data:function(a,b,c){return X.access(a,b,c)},removeData:function(a,b){X.remove(a,b)},_data:function(a,b,c){return W.access(a,b,c)},_removeData:function(a,b){W.remove(a,b)}}),r.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=X.get(f),1===f.nodeType&&!W.get(f,"hasDataAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=r.camelCase(d.slice(5)),_(f,d,e[d])));W.set(f,"hasDataAttrs",!0)}return e}return"object"==typeof a?this.each(function(){X.set(this,a)}):T(this,function(b){var c;if(f&&void 0===b){if(c=X.get(f,a),void 0!==c)return c;if(c=_(f,a),void 0!==c)return c}else this.each(function(){X.set(this,a,b)})},null,b,arguments.length>1,null,!0)},removeData:function(a){return this.each(function(){X.remove(this,a)})}}),r.extend({queue:function(a,b,c){var d;if(a)return b=(b||"fx")+"queue",d=W.get(a,b),c&&(!d||Array.isArray(c)?d=W.access(a,b,r.makeArray(c)):d.push(c)),d||[]},dequeue:function(a,b){b=b||
"fx";var c=r.queue(a,b),d=c.length,e=c.shift(),f=r._queueHooks(a,b),g=function(){r.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return W.get(a,c)||W.access(a,c,{empty:r.Callbacks("once memory").add(function(){W.remove(a,[b+"queue",c])})})}}),r.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.length<c?r.queue(this[0],a):void 0===b?this:this.each(function(){var c=r.queue(this,a,b);r._queueHooks(this,a),"fx"===a&&"inprogress"!==c[0]&&r.dequeue(this,a)})},dequeue:function(a){return this.each(function(){r.dequeue(this,a)})},clearQueue:function(a){return this.queue(a||"fx",[])},promise:function(a,b){var c,d=1,e=r.Deferred(),f=this,g=this.length,h=function(){--d||e.resolveWith(f,[f])};"string"!=typeof a&&(b=a,a=void 0),a=a||"fx";while(g--)c=W.get(f[g],a+"queueHooks"),c&&c.empty&&(d++,c.empty.add(h));return h
(),e.promise(b)}});var aa=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,ba=new RegExp("^(?:([+-])=|)("+aa+")([a-z%]*)$","i"),ca=["Top","Right","Bottom","Left"],da=function(a,b){return a=b||a,"none"===a.style.display||""===a.style.display&&r.contains(a.ownerDocument,a)&&"none"===r.css(a,"display")},ea=function(a,b,c,d){var e,f,g={};for(f in b)g[f]=a.style[f],a.style[f]=b[f];e=c.apply(a,d||[]);for(f in b)a.style[f]=g[f];return e};function fa(a,b,c,d){var e,f=1,g=20,h=d?function(){return d.cur()}:function(){return r.css(a,b,"")},i=h(),j=c&&c[3]||(r.cssNumber[b]?"":"px"),k=(r.cssNumber[b]||"px"!==j&&+i)&&ba.exec(r.css(a,b));if(k&&k[3]!==j){j=j||k[3],c=c||[],k=+i||1;do f=f||".5",k/=f,r.style(a,b,k+j);while(f!==(f=h()/i)&&1!==f&&--g)}return c&&(k=+k||+i||0,e=c[1]?k+(c[1]+1)*c[2]:+c[2],d&&(d.unit=j,d.start=k,d.end=e)),e}var ga={};function ha(a){var b,c=a.ownerDocument,d=a.nodeName,e=ga[d];return e?e:(b=c.body.appendChild(c.createElement(d)),e=r.css(b,"display"),b.parentNode.removeChild(b),"
none"===e&&(e="block"),ga[d]=e,e)}function ia(a,b){for(var c,d,e=[],f=0,g=a.length;f<g;f++)d=a[f],d.style&&(c=d.style.display,b?("none"===c&&(e[f]=W.get(d,"display")||null,e[f]||(d.style.display="")),""===d.style.display&&da(d)&&(e[f]=ha(d))):"none"!==c&&(e[f]="none",W.set(d,"display",c)));for(f=0;f<g;f++)null!=e[f]&&(a[f].style.display=e[f]);return a}r.fn.extend({show:function(){return ia(this,!0)},hide:function(){return ia(this)},toggle:function(a){return"boolean"==typeof a?a?this.show():this.hide():this.each(function(){da(this)?r(this).show():r(this).hide()})}});var ja=/^(?:checkbox|radio)$/i,ka=/<([a-z][^\/\0>\x20\t\r\n\f]+)/i,la=/^$|\/(?:java|ecma)script/i,ma={option:[1,"<select multiple='multiple'>","</select>"],thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};ma.optgroup=ma.option,ma.tbody=ma.tfoot=ma.colgroup=ma.caption=ma.thead,m
a.th=ma.td;function na(a,b){var c;return c="undefined"!=typeof a.getElementsByTagName?a.getElementsByTagName(b||"*"):"undefined"!=typeof a.querySelectorAll?a.querySelectorAll(b||"*"):[],void 0===b||b&&B(a,b)?r.merge([a],c):c}function oa(a,b){for(var c=0,d=a.length;c<d;c++)W.set(a[c],"globalEval",!b||W.get(b[c],"globalEval"))}var pa=/<|&#?\w+;/;function qa(a,b,c,d,e){for(var f,g,h,i,j,k,l=b.createDocumentFragment(),m=[],n=0,o=a.length;n<o;n++)if(f=a[n],f||0===f)if("object"===r.type(f))r.merge(m,f.nodeType?[f]:f);else if(pa.test(f)){g=g||l.appendChild(b.createElement("div")),h=(ka.exec(f)||["",""])[1].toLowerCase(),i=ma[h]||ma._default,g.innerHTML=i[1]+r.htmlPrefilter(f)+i[2],k=i[0];while(k--)g=g.lastChild;r.merge(m,g.childNodes),g=l.firstChild,g.textContent=""}else m.push(b.createTextNode(f));l.textContent="",n=0;while(f=m[n++])if(d&&r.inArray(f,d)>-1)e&&e.push(f);else if(j=r.contains(f.ownerDocument,f),g=na(l.appendChild(f),"script"),j&&oa(g),c){k=0;while(f=g[k++])la.test(f.type||""
)&&c.push(f)}return l}!function(){var a=d.createDocumentFragment(),b=a.appendChild(d.createElement("div")),c=d.createElement("input");c.setAttribute("type","radio"),c.setAttribute("checked","checked"),c.setAttribute("name","t"),b.appendChild(c),o.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,b.innerHTML="<textarea>x</textarea>",o.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue}();var ra=d.documentElement,sa=/^key/,ta=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,ua=/^([^.]*)(?:\.(.+)|)/;function va(){return!0}function wa(){return!1}function xa(){try{return d.activeElement}catch(a){}}function ya(a,b,c,d,e,f){var g,h;if("object"==typeof b){"string"!=typeof c&&(d=d||c,c=void 0);for(h in b)ya(a,h,c,d,b[h],f);return a}if(null==d&&null==e?(e=c,d=c=void 0):null==e&&("string"==typeof c?(e=d,d=void 0):(e=d,d=c,c=void 0)),e===!1)e=wa;else if(!e)return a;return 1===f&&(g=e,e=function(a){return r().off(a),g.apply(this,arguments)},e.guid=g.guid||(g.guid=r.guid++)),a.each(
function(){r.event.add(this,b,e,d,c)})}r.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,n,o,p,q=W.get(a);if(q){c.handler&&(f=c,c=f.handler,e=f.selector),e&&r.find.matchesSelector(ra,e),c.guid||(c.guid=r.guid++),(i=q.events)||(i=q.events={}),(g=q.handle)||(g=q.handle=function(b){return"undefined"!=typeof r&&r.event.triggered!==b.type?r.event.dispatch.apply(a,arguments):void 0}),b=(b||"").match(L)||[""],j=b.length;while(j--)h=ua.exec(b[j])||[],n=p=h[1],o=(h[2]||"").split(".").sort(),n&&(l=r.event.special[n]||{},n=(e?l.delegateType:l.bindType)||n,l=r.event.special[n]||{},k=r.extend({type:n,origType:p,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&r.expr.match.needsContext.test(e),namespace:o.join(".")},f),(m=i[n])||(m=i[n]=[],m.delegateCount=0,l.setup&&l.setup.call(a,d,o,g)!==!1||a.addEventListener&&a.addEventListener(n,g)),l.add&&(l.add.call(a,k),k.handler.guid||(k.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,k):m.push(k),r.event.global[n]=!0)}},remove:f
unction(a,b,c,d,e){var f,g,h,i,j,k,l,m,n,o,p,q=W.hasData(a)&&W.get(a);if(q&&(i=q.events)){b=(b||"").match(L)||[""],j=b.length;while(j--)if(h=ua.exec(b[j])||[],n=p=h[1],o=(h[2]||"").split(".").sort(),n){l=r.event.special[n]||{},n=(d?l.delegateType:l.bindType)||n,m=i[n]||[],h=h[2]&&new RegExp("(^|\\.)"+o.join("\\.(?:.*\\.|)")+"(\\.|$)"),g=f=m.length;while(f--)k=m[f],!e&&p!==k.origType||c&&c.guid!==k.guid||h&&!h.test(k.namespace)||d&&d!==k.selector&&("**"!==d||!k.selector)||(m.splice(f,1),k.selector&&m.delegateCount--,l.remove&&l.remove.call(a,k));g&&!m.length&&(l.teardown&&l.teardown.call(a,o,q.handle)!==!1||r.removeEvent(a,n,q.handle),delete i[n])}else for(n in i)r.event.remove(a,n+b[j],c,d,!0);r.isEmptyObject(i)&&W.remove(a,"handle events")}},dispatch:function(a){var b=r.event.fix(a),c,d,e,f,g,h,i=new Array(arguments.length),j=(W.get(this,"events")||{})[b.type]||[],k=r.event.special[b.type]||{};for(i[0]=b,c=1;c<arguments.length;c++)i[c]=arguments[c];if(b.delegateTarget=this,!k.preDi
spatch||k.preDispatch.call(this,b)!==!1){h=r.event.handlers.call(this,b,j),c=0;while((f=h[c++])&&!b.isPropagationStopped()){b.currentTarget=f.elem,d=0;while((g=f.handlers[d++])&&!b.isImmediatePropagationStopped())b.rnamespace&&!b.rnamespace.test(g.namespace)||(b.handleObj=g,b.data=g.data,e=((r.event.special[g.origType]||{}).handle||g.handler).apply(f.elem,i),void 0!==e&&(b.result=e)===!1&&(b.preventDefault(),b.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,b),b.result}},handlers:function(a,b){var c,d,e,f,g,h=[],i=b.delegateCount,j=a.target;if(i&&j.nodeType&&!("click"===a.type&&a.button>=1))for(;j!==this;j=j.parentNode||this)if(1===j.nodeType&&("click"!==a.type||j.disabled!==!0)){for(f=[],g={},c=0;c<i;c++)d=b[c],e=d.selector+" ",void 0===g[e]&&(g[e]=d.needsContext?r(e,this).index(j)>-1:r.find(e,this,null,[j]).length),g[e]&&f.push(d);f.length&&h.push({elem:j,handlers:f})}return j=this,i<b.length&&h.push({elem:j,handlers:b.slice(i)}),h},addProp:function(a,b){Object
.defineProperty(r.Event.prototype,a,{enumerable:!0,configurable:!0,get:r.isFunction(b)?function(){if(this.originalEvent)return b(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[a]},set:function(b){Object.defineProperty(this,a,{enumerable:!0,configurable:!0,writable:!0,value:b})}})},fix:function(a){return a[r.expando]?a:new r.Event(a)},special:{load:{noBubble:!0},focus:{trigger:function(){if(this!==xa()&&this.focus)return this.focus(),!1},delegateType:"focusin"},blur:{trigger:function(){if(this===xa()&&this.blur)return this.blur(),!1},delegateType:"focusout"},click:{trigger:function(){if("checkbox"===this.type&&this.click&&B(this,"input"))return this.click(),!1},_default:function(a){return B(a.target,"a")}},beforeunload:{postDispatch:function(a){void 0!==a.result&&a.originalEvent&&(a.originalEvent.returnValue=a.result)}}}},r.removeEvent=function(a,b,c){a.removeEventListener&&a.removeEventListener(b,c)},r.Event=function(a,b){return this instanceof r.Eve
nt?(a&&a.type?(this.originalEvent=a,this.type=a.type,this.isDefaultPrevented=a.defaultPrevented||void 0===a.defaultPrevented&&a.returnValue===!1?va:wa,this.target=a.target&&3===a.target.nodeType?a.target.parentNode:a.target,this.currentTarget=a.currentTarget,this.relatedTarget=a.relatedTarget):this.type=a,b&&r.extend(this,b),this.timeStamp=a&&a.timeStamp||r.now(),void(this[r.expando]=!0)):new r.Event(a,b)},r.Event.prototype={constructor:r.Event,isDefaultPrevented:wa,isPropagationStopped:wa,isImmediatePropagationStopped:wa,isSimulated:!1,preventDefault:function(){var a=this.originalEvent;this.isDefaultPrevented=va,a&&!this.isSimulated&&a.preventDefault()},stopPropagation:function(){var a=this.originalEvent;this.isPropagationStopped=va,a&&!this.isSimulated&&a.stopPropagation()},stopImmediatePropagation:function(){var a=this.originalEvent;this.isImmediatePropagationStopped=va,a&&!this.isSimulated&&a.stopImmediatePropagation(),this.stopPropagation()}},r.each({altKey:!0,bubbles:!0,cancel
able:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:function(a){var b=a.button;return null==a.which&&sa.test(a.type)?null!=a.charCode?a.charCode:a.keyCode:!a.which&&void 0!==b&&ta.test(a.type)?1&b?1:2&b?3:4&b?2:0:a.which}},r.event.addProp),r.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(a,b){r.event.special[a]={delegateType:b,bindType:b,handle:function(a){var c,d=this,e=a.relatedTarget,f=a.handleObj;return e&&(e===d||r.contains(d,e))||(a.type=f.origType,c=f.handler.apply(this,arguments),a.type=b),c}}}),r.fn.extend({on:function(a,b,c,d){return ya(this,a,b,c,d)},one:function(a,b,c,d){return ya(this,a,b,c,d,1)},off:function(a,b,c){var d,e;if(a&&a.preventDefault&&a
.handleObj)return d=a.handleObj,r(a.delegateTarget).off(d.namespace?d.origType+"."+d.namespace:d.origType,d.selector,d.handler),this;if("object"==typeof a){for(e in a)this.off(e,b,a[e]);return this}return b!==!1&&"function"!=typeof b||(c=b,b=void 0),c===!1&&(c=wa),this.each(function(){r.event.remove(this,a,c,b)})}});var za=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([a-z][^\/\0>\x20\t\r\n\f]*)[^>]*)\/>/gi,Aa=/<script|<style|<link/i,Ba=/checked\s*(?:[^=]|=\s*.checked.)/i,Ca=/^true\/(.*)/,Da=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function Ea(a,b){return B(a,"table")&&B(11!==b.nodeType?b:b.firstChild,"tr")?r(">tbody",a)[0]||a:a}function Fa(a){return a.type=(null!==a.getAttribute("type"))+"/"+a.type,a}function Ga(a){var b=Ca.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function Ha(a,b){var c,d,e,f,g,h,i,j;if(1===b.nodeType){if(W.hasData(a)&&(f=W.access(a),g=W.set(b,f),j=f.events)){delete g.handle,g.events={};for(e in j)for(c=0,d=j[e].length;c<d;c++)r.event
.add(b,e,j[e][c])}X.hasData(a)&&(h=X.access(a),i=r.extend({},h),X.set(b,i))}}function Ia(a,b){var c=b.nodeName.toLowerCase();"input"===c&&ja.test(a.type)?b.checked=a.checked:"input"!==c&&"textarea"!==c||(b.defaultValue=a.defaultValue)}function Ja(a,b,c,d){b=g.apply([],b);var e,f,h,i,j,k,l=0,m=a.length,n=m-1,q=b[0],s=r.isFunction(q);if(s||m>1&&"string"==typeof q&&!o.checkClone&&Ba.test(q))return a.each(function(e){var f=a.eq(e);s&&(b[0]=q.call(this,e,f.html())),Ja(f,b,c,d)});if(m&&(e=qa(b,a[0].ownerDocument,!1,a,d),f=e.firstChild,1===e.childNodes.length&&(e=f),f||d)){for(h=r.map(na(e,"script"),Fa),i=h.length;l<m;l++)j=e,l!==n&&(j=r.clone(j,!0,!0),i&&r.merge(h,na(j,"script"))),c.call(a[l],j,l);if(i)for(k=h[h.length-1].ownerDocument,r.map(h,Ga),l=0;l<i;l++)j=h[l],la.test(j.type||"")&&!W.access(j,"globalEval")&&r.contains(k,j)&&(j.src?r._evalUrl&&r._evalUrl(j.src):p(j.textContent.replace(Da,""),k))}return a}function Ka(a,b,c){for(var d,e=b?r.filter(b,a):a,f=0;null!=(d=e[f]);f++)c||1!==d
.nodeType||r.cleanData(na(d)),d.parentNode&&(c&&r.contains(d.ownerDocument,d)&&oa(na(d,"script")),d.parentNode.removeChild(d));return a}r.extend({htmlPrefilter:function(a){return a.replace(za,"<$1></$2>")},clone:function(a,b,c){var d,e,f,g,h=a.cloneNode(!0),i=r.contains(a.ownerDocument,a);if(!(o.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||r.isXMLDoc(a)))for(g=na(h),f=na(a),d=0,e=f.length;d<e;d++)Ia(f[d],g[d]);if(b)if(c)for(f=f||na(a),g=g||na(h),d=0,e=f.length;d<e;d++)Ha(f[d],g[d]);else Ha(a,h);return g=na(h,"script"),g.length>0&&oa(g,!i&&na(a,"script")),h},cleanData:function(a){for(var b,c,d,e=r.event.special,f=0;void 0!==(c=a[f]);f++)if(U(c)){if(b=c[W.expando]){if(b.events)for(d in b.events)e[d]?r.event.remove(c,d):r.removeEvent(c,d,b.handle);c[W.expando]=void 0}c[X.expando]&&(c[X.expando]=void 0)}}}),r.fn.extend({detach:function(a){return Ka(this,a,!0)},remove:function(a){return Ka(this,a)},text:function(a){return T(this,function(a){return void 0===a?r.text(this):this.empty()
.each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=a)})},null,a,arguments.length)},append:function(){return Ja(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Ea(this,a);b.appendChild(a)}})},prepend:function(){return Ja(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Ea(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return Ja(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return Ja(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},empty:function(){for(var a,b=0;null!=(a=this[b]);b++)1===a.nodeType&&(r.cleanData(na(a,!1)),a.textContent="");return this},clone:function(a,b){return a=null!=a&&a,b=null==b?a:b,this.map(function(){return r.clone(this,a,b)})},html:function(a){return T(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0=
==a&&1===b.nodeType)return b.innerHTML;if("string"==typeof a&&!Aa.test(a)&&!ma[(ka.exec(a)||["",""])[1].toLowerCase()]){a=r.htmlPrefilter(a);try{for(;c<d;c++)b=this[c]||{},1===b.nodeType&&(r.cleanData(na(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=[];return Ja(this,arguments,function(b){var c=this.parentNode;r.inArray(this,a)<0&&(r.cleanData(na(this)),c&&c.replaceChild(b,this))},a)}}),r.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){r.fn[a]=function(a){for(var c,d=[],e=r(a),f=e.length-1,g=0;g<=f;g++)c=g===f?this:this.clone(!0),r(e[g])[b](c),h.apply(d,c.get());return this.pushStack(d)}});var La=/^margin/,Ma=new RegExp("^("+aa+")(?!px)[a-z%]+$","i"),Na=function(b){var c=b.ownerDocument.defaultView;return c&&c.opener||(c=a),c.getComputedStyle(b)};!function(){function b(){if(i){i.style.cssText="box-sizing:border-box;position:relative;d
isplay:block;margin:auto;border:1px;padding:1px;top:1%;width:50%",i.innerHTML="",ra.appendChild(h);var b=a.getComputedStyle(i);c="1%"!==b.top,g="2px"===b.marginLeft,e="4px"===b.width,i.style.marginRight="50%",f="4px"===b.marginRight,ra.removeChild(h),i=null}}var c,e,f,g,h=d.createElement("div"),i=d.createElement("div");i.style&&(i.style.backgroundClip="content-box",i.cloneNode(!0).style.backgroundClip="",o.clearCloneStyle="content-box"===i.style.backgroundClip,h.style.cssText="border:0;width:8px;height:0;top:0;left:-9999px;padding:0;margin-top:1px;position:absolute",h.appendChild(i),r.extend(o,{pixelPosition:function(){return b(),c},boxSizingReliable:function(){return b(),e},pixelMarginRight:function(){return b(),f},reliableMarginLeft:function(){return b(),g}}))}();function Oa(a,b,c){var d,e,f,g,h=a.style;return c=c||Na(a),c&&(g=c.getPropertyValue(b)||c[b],""!==g||r.contains(a.ownerDocument,a)||(g=r.style(a,b)),!o.pixelMarginRight()&&Ma.test(g)&&La.test(b)&&(d=h.width,e=h.minWidth,f
=h.maxWidth,h.minWidth=h.maxWidth=h.width=g,g=c.width,h.width=d,h.minWidth=e,h.maxWidth=f)),void 0!==g?g+"":g}function Pa(a,b){return{get:function(){return a()?void delete this.get:(this.get=b).apply(this,arguments)}}}var Qa=/^(none|table(?!-c[ea]).+)/,Ra=/^--/,Sa={position:"absolute",visibility:"hidden",display:"block"},Ta={letterSpacing:"0",fontWeight:"400"},Ua=["Webkit","Moz","ms"],Va=d.createElement("div").style;function Wa(a){if(a in Va)return a;var b=a[0].toUpperCase()+a.slice(1),c=Ua.length;while(c--)if(a=Ua[c]+b,a in Va)return a}function Xa(a){var b=r.cssProps[a];return b||(b=r.cssProps[a]=Wa(a)||a),b}function Ya(a,b,c){var d=ba.exec(b);return d?Math.max(0,d[2]-(c||0))+(d[3]||"px"):b}function Za(a,b,c,d,e){var f,g=0;for(f=c===(d?"border":"content")?4:"width"===b?1:0;f<4;f+=2)"margin"===c&&(g+=r.css(a,c+ca[f],!0,e)),d?("content"===c&&(g-=r.css(a,"padding"+ca[f],!0,e)),"margin"!==c&&(g-=r.css(a,"border"+ca[f]+"Width",!0,e))):(g+=r.css(a,"padding"+ca[f],!0,e),"padding"!==c&&(g+
=r.css(a,"border"+ca[f]+"Width",!0,e)));return g}function $a(a,b,c){var d,e=Na(a),f=Oa(a,b,e),g="border-box"===r.css(a,"boxSizing",!1,e);return Ma.test(f)?f:(d=g&&(o.boxSizingReliable()||f===a.style[b]),"auto"===f&&(f=a["offset"+b[0].toUpperCase()+b.slice(1)]),f=parseFloat(f)||0,f+Za(a,b,c||(g?"border":"content"),d,e)+"px")}r.extend({cssHooks:{opacity:{get:function(a,b){if(b){var c=Oa(a,"opacity");return""===c?"1":c}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":"cssFloat"},style:function(a,b,c,d){if(a&&3!==a.nodeType&&8!==a.nodeType&&a.style){var e,f,g,h=r.camelCase(b),i=Ra.test(b),j=a.style;return i||(b=Xa(h)),g=r.cssHooks[b]||r.cssHooks[h],void 0===c?g&&"get"in g&&void 0!==(e=g.get(a,!1,d))?e:j[b]:(f=typeof c,"string"===f&&(e=ba.exec(c))&&e[1]&&(c=fa(a,b,e),f="number"),null!=c&&c===c&&("number"===f&&(c+=e&&e[3]||(r.cssNumber[
h]?"":"px")),o.clearCloneStyle||""!==c||0!==b.indexOf("background")||(j[b]="inherit"),g&&"set"in g&&void 0===(c=g.set(a,c,d))||(i?j.setProperty(b,c):j[b]=c)),void 0)}},css:function(a,b,c,d){var e,f,g,h=r.camelCase(b),i=Ra.test(b);return i||(b=Xa(h)),g=r.cssHooks[b]||r.cssHooks[h],g&&"get"in g&&(e=g.get(a,!0,c)),void 0===e&&(e=Oa(a,b,d)),"normal"===e&&b in Ta&&(e=Ta[b]),""===c||c?(f=parseFloat(e),c===!0||isFinite(f)?f||0:e):e}}),r.each(["height","width"],function(a,b){r.cssHooks[b]={get:function(a,c,d){if(c)return!Qa.test(r.css(a,"display"))||a.getClientRects().length&&a.getBoundingClientRect().width?$a(a,b,d):ea(a,Sa,function(){return $a(a,b,d)})},set:function(a,c,d){var e,f=d&&Na(a),g=d&&Za(a,b,d,"border-box"===r.css(a,"boxSizing",!1,f),f);return g&&(e=ba.exec(c))&&"px"!==(e[3]||"px")&&(a.style[b]=c,c=r.css(a,b)),Ya(a,c,g)}}}),r.cssHooks.marginLeft=Pa(o.reliableMarginLeft,function(a,b){if(b)return(parseFloat(Oa(a,"marginLeft"))||a.getBoundingClientRect().left-ea(a,{marginLeft:0},fu
nction(){return a.getBoundingClientRect().left}))+"px"}),r.each({margin:"",padding:"",border:"Width"},function(a,b){r.cssHooks[a+b]={expand:function(c){for(var d=0,e={},f="string"==typeof c?c.split(" "):[c];d<4;d++)e[a+ca[d]+b]=f[d]||f[d-2]||f[0];return e}},La.test(a)||(r.cssHooks[a+b].set=Ya)}),r.fn.extend({css:function(a,b){return T(this,function(a,b,c){var d,e,f={},g=0;if(Array.isArray(b)){for(d=Na(a),e=b.length;g<e;g++)f[b[g]]=r.css(a,b[g],!1,d);return f}return void 0!==c?r.style(a,b,c):r.css(a,b)},a,b,arguments.length>1)}});function _a(a,b,c,d,e){return new _a.prototype.init(a,b,c,d,e)}r.Tween=_a,_a.prototype={constructor:_a,init:function(a,b,c,d,e,f){this.elem=a,this.prop=c,this.easing=e||r.easing._default,this.options=b,this.start=this.now=this.cur(),this.end=d,this.unit=f||(r.cssNumber[c]?"":"px")},cur:function(){var a=_a.propHooks[this.prop];return a&&a.get?a.get(this):_a.propHooks._default.get(this)},run:function(a){var b,c=_a.propHooks[this.prop];return this.options.durat
ion?this.pos=b=r.easing[this.easing](a,this.options.duration*a,0,1,this.options.duration):this.pos=b=a,this.now=(this.end-this.start)*b+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),c&&c.set?c.set(this):_a.propHooks._default.set(this),this}},_a.prototype.init.prototype=_a.prototype,_a.propHooks={_default:{get:function(a){var b;return 1!==a.elem.nodeType||null!=a.elem[a.prop]&&null==a.elem.style[a.prop]?a.elem[a.prop]:(b=r.css(a.elem,a.prop,""),b&&"auto"!==b?b:0)},set:function(a){r.fx.step[a.prop]?r.fx.step[a.prop](a):1!==a.elem.nodeType||null==a.elem.style[r.cssProps[a.prop]]&&!r.cssHooks[a.prop]?a.elem[a.prop]=a.now:r.style(a.elem,a.prop,a.now+a.unit)}}},_a.propHooks.scrollTop=_a.propHooks.scrollLeft={set:function(a){a.elem.nodeType&&a.elem.parentNode&&(a.elem[a.prop]=a.now)}},r.easing={linear:function(a){return a},swing:function(a){return.5-Math.cos(a*Math.PI)/2},_default:"swing"},r.fx=_a.prototype.init,r.fx.step={};var ab,bb,cb=/^(?:toggle|show|hid
e)$/,db=/queueHooks$/;function eb(){bb&&(d.hidden===!1&&a.requestAnimationFrame?a.requestAnimationFrame(eb):a.setTimeout(eb,r.fx.interval),r.fx.tick())}function fb(){return a.setTimeout(function(){ab=void 0}),ab=r.now()}function gb(a,b){var c,d=0,e={height:a};for(b=b?1:0;d<4;d+=2-b)c=ca[d],e["margin"+c]=e["padding"+c]=a;return b&&(e.opacity=e.width=a),e}function hb(a,b,c){for(var d,e=(kb.tweeners[b]||[]).concat(kb.tweeners["*"]),f=0,g=e.length;f<g;f++)if(d=e[f].call(c,b,a))return d}function ib(a,b,c){var d,e,f,g,h,i,j,k,l="width"in b||"height"in b,m=this,n={},o=a.style,p=a.nodeType&&da(a),q=W.get(a,"fxshow");c.queue||(g=r._queueHooks(a,"fx"),null==g.unqueued&&(g.unqueued=0,h=g.empty.fire,g.empty.fire=function(){g.unqueued||h()}),g.unqueued++,m.always(function(){m.always(function(){g.unqueued--,r.queue(a,"fx").length||g.empty.fire()})}));for(d in b)if(e=b[d],cb.test(e)){if(delete b[d],f=f||"toggle"===e,e===(p?"hide":"show")){if("show"!==e||!q||void 0===q[d])continue;p=!0}n[d]=q&&q[d]
||r.style(a,d)}if(i=!r.isEmptyObject(b),i||!r.isEmptyObject(n)){l&&1===a.nodeType&&(c.overflow=[o.overflow,o.overflowX,o.overflowY],j=q&&q.display,null==j&&(j=W.get(a,"display")),k=r.css(a,"display"),"none"===k&&(j?k=j:(ia([a],!0),j=a.style.display||j,k=r.css(a,"display"),ia([a]))),("inline"===k||"inline-block"===k&&null!=j)&&"none"===r.css(a,"float")&&(i||(m.done(function(){o.display=j}),null==j&&(k=o.display,j="none"===k?"":k)),o.display="inline-block")),c.overflow&&(o.overflow="hidden",m.always(function(){o.overflow=c.overflow[0],o.overflowX=c.overflow[1],o.overflowY=c.overflow[2]})),i=!1;for(d in n)i||(q?"hidden"in q&&(p=q.hidden):q=W.access(a,"fxshow",{display:j}),f&&(q.hidden=!p),p&&ia([a],!0),m.done(function(){p||ia([a]),W.remove(a,"fxshow");for(d in n)r.style(a,d,n[d])})),i=hb(p?q[d]:0,d,m),d in q||(q[d]=i.start,p&&(i.end=i.start,i.start=0))}}function jb(a,b){var c,d,e,f,g;for(c in a)if(d=r.camelCase(c),e=b[d],f=a[c],Array.isArray(f)&&(e=f[1],f=a[c]=f[0]),c!==d&&(a[d]=f,dele
te a[c]),g=r.cssHooks[d],g&&"expand"in g){f=g.expand(f),delete a[d];for(c in f)c in a||(a[c]=f[c],b[c]=e)}else b[d]=e}function kb(a,b,c){var d,e,f=0,g=kb.prefilters.length,h=r.Deferred().always(function(){delete i.elem}),i=function(){if(e)return!1;for(var b=ab||fb(),c=Math.max(0,j.startTime+j.duration-b),d=c/j.duration||0,f=1-d,g=0,i=j.tweens.length;g<i;g++)j.tweens[g].run(f);return h.notifyWith(a,[j,f,c]),f<1&&i?c:(i||h.notifyWith(a,[j,1,0]),h.resolveWith(a,[j]),!1)},j=h.promise({elem:a,props:r.extend({},b),opts:r.extend(!0,{specialEasing:{},easing:r.easing._default},c),originalProperties:b,originalOptions:c,startTime:ab||fb(),duration:c.duration,tweens:[],createTween:function(b,c){var d=r.Tween(a,j.opts,b,c,j.opts.specialEasing[b]||j.opts.easing);return j.tweens.push(d),d},stop:function(b){var c=0,d=b?j.tweens.length:0;if(e)return this;for(e=!0;c<d;c++)j.tweens[c].run(1);return b?(h.notifyWith(a,[j,1,0]),h.resolveWith(a,[j,b])):h.rejectWith(a,[j,b]),this}}),k=j.props;for(jb(k,j.op
ts.specialEasing);f<g;f++)if(d=kb.prefilters[f].call(j,a,k,j.opts))return r.isFunction(d.stop)&&(r._queueHooks(j.elem,j.opts.queue).stop=r.proxy(d.stop,d)),d;return r.map(k,hb,j),r.isFunction(j.opts.start)&&j.opts.start.call(a,j),j.progress(j.opts.progress).done(j.opts.done,j.opts.complete).fail(j.opts.fail).always(j.opts.always),r.fx.timer(r.extend(i,{elem:a,anim:j,queue:j.opts.queue})),j}r.Animation=r.extend(kb,{tweeners:{"*":[function(a,b){var c=this.createTween(a,b);return fa(c.elem,a,ba.exec(b),c),c}]},tweener:function(a,b){r.isFunction(a)?(b=a,a=["*"]):a=a.match(L);for(var c,d=0,e=a.length;d<e;d++)c=a[d],kb.tweeners[c]=kb.tweeners[c]||[],kb.tweeners[c].unshift(b)},prefilters:[ib],prefilter:function(a,b){b?kb.prefilters.unshift(a):kb.prefilters.push(a)}}),r.speed=function(a,b,c){var d=a&&"object"==typeof a?r.extend({},a):{complete:c||!c&&b||r.isFunction(a)&&a,duration:a,easing:c&&b||b&&!r.isFunction(b)&&b};return r.fx.off?d.duration=0:"number"!=typeof d.duration&&(d.duration in
r.fx.speeds?d.duration=r.fx.speeds[d.duration]:d.duration=r.fx.speeds._default),null!=d.queue&&d.queue!==!0||(d.queue="fx"),d.old=d.complete,d.complete=function(){r.isFunction(d.old)&&d.old.call(this),d.queue&&r.dequeue(this,d.queue)},d},r.fn.extend({fadeTo:function(a,b,c,d){return this.filter(da).css("opacity",0).show().end().animate({opacity:b},a,c,d)},animate:function(a,b,c,d){var e=r.isEmptyObject(a),f=r.speed(b,c,d),g=function(){var b=kb(this,r.extend({},a),f);(e||W.get(this,"finish"))&&b.stop(!0)};return g.finish=g,e||f.queue===!1?this.each(g):this.queue(f.queue,g)},stop:function(a,b,c){var d=function(a){var b=a.stop;delete a.stop,b(c)};return"string"!=typeof a&&(c=b,b=a,a=void 0),b&&a!==!1&&this.queue(a||"fx",[]),this.each(function(){var b=!0,e=null!=a&&a+"queueHooks",f=r.timers,g=W.get(this);if(e)g[e]&&g[e].stop&&d(g[e]);else for(e in g)g[e]&&g[e].stop&&db.test(e)&&d(g[e]);for(e=f.length;e--;)f[e].elem!==this||null!=a&&f[e].queue!==a||(f[e].anim.stop(c),b=!1,f.splice(e,1));
!b&&c||r.dequeue(this,a)})},finish:function(a){return a!==!1&&(a=a||"fx"),this.each(function(){var b,c=W.get(this),d=c[a+"queue"],e=c[a+"queueHooks"],f=r.timers,g=d?d.length:0;for(c.finish=!0,r.queue(this,a,[]),e&&e.stop&&e.stop.call(this,!0),b=f.length;b--;)f[b].elem===this&&f[b].queue===a&&(f[b].anim.stop(!0),f.splice(b,1));for(b=0;b<g;b++)d[b]&&d[b].finish&&d[b].finish.call(this);delete c.finish})}}),r.each(["toggle","show","hide"],function(a,b){var c=r.fn[b];r.fn[b]=function(a,d,e){return null==a||"boolean"==typeof a?c.apply(this,arguments):this.animate(gb(b,!0),a,d,e)}}),r.each({slideDown:gb("show"),slideUp:gb("hide"),slideToggle:gb("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(a,b){r.fn[a]=function(a,c,d){return this.animate(b,a,c,d)}}),r.timers=[],r.fx.tick=function(){var a,b=0,c=r.timers;for(ab=r.now();b<c.length;b++)a=c[b],a()||c[b]!==a||c.splice(b--,1);c.length||r.fx.stop(),ab=void 0},r.fx.timer=function(a){r.timers.push
(a),r.fx.start()},r.fx.interval=13,r.fx.start=function(){bb||(bb=!0,eb())},r.fx.stop=function(){bb=null},r.fx.speeds={slow:600,fast:200,_default:400},r.fn.delay=function(b,c){return b=r.fx?r.fx.speeds[b]||b:b,c=c||"fx",this.queue(c,function(c,d){var e=a.setTimeout(c,b);d.stop=function(){a.clearTimeout(e)}})},function(){var a=d.createElement("input"),b=d.createElement("select"),c=b.appendChild(d.createElement("option"));a.type="checkbox",o.checkOn=""!==a.value,o.optSelected=c.selected,a=d.createElement("input"),a.value="t",a.type="radio",o.radioValue="t"===a.value}();var lb,mb=r.expr.attrHandle;r.fn.extend({attr:function(a,b){return T(this,r.attr,a,b,arguments.length>1)},removeAttr:function(a){return this.each(function(){r.removeAttr(this,a)})}}),r.extend({attr:function(a,b,c){var d,e,f=a.nodeType;if(3!==f&&8!==f&&2!==f)return"undefined"==typeof a.getAttribute?r.prop(a,b,c):(1===f&&r.isXMLDoc(a)||(e=r.attrHooks[b.toLowerCase()]||(r.expr.match.bool.test(b)?lb:void 0)),void 0!==c?null=
==c?void r.removeAttr(a,b):e&&"set"in e&&void 0!==(d=e.set(a,c,b))?d:(a.setAttribute(b,c+""),c):e&&"get"in e&&null!==(d=e.get(a,b))?d:(d=r.find.attr(a,b),
-null==d?void 0:d))},attrHooks:{type:{set:function(a,b){if(!o.radioValue&&"radio"===b&&B(a,"input")){var c=a.value;return a.setAttribute("type",b),c&&(a.value=c),b}}}},removeAttr:function(a,b){var c,d=0,e=b&&b.match(L);if(e&&1===a.nodeType)while(c=e[d++])a.removeAttribute(c)}}),lb={set:function(a,b,c){return b===!1?r.removeAttr(a,c):a.setAttribute(c,c),c}},r.each(r.expr.match.bool.source.match(/\w+/g),function(a,b){var c=mb[b]||r.find.attr;mb[b]=function(a,b,d){var e,f,g=b.toLowerCase();return d||(f=mb[g],mb[g]=e,e=null!=c(a,b,d)?g:null,mb[g]=f),e}});var nb=/^(?:input|select|textarea|button)$/i,ob=/^(?:a|area)$/i;r.fn.extend({prop:function(a,b){return T(this,r.prop,a,b,arguments.length>1)},removeProp:function(a){return this.each(function(){delete this[r.propFix[a]||a]})}}),r.extend({prop:function(a,b,c){var d,e,f=a.nodeType;if(3!==f&&8!==f&&2!==f)return 1===f&&r.isXMLDoc(a)||(b=r.propFix[b]||b,e=r.propHooks[b]),void 0!==c?e&&"set"in e&&void 0!==(d=e.set(a,c,b))?d:a[b]=c:e&&"get"in e&
&null!==(d=e.get(a,b))?d:a[b]},propHooks:{tabIndex:{get:function(a){var b=r.find.attr(a,"tabindex");return b?parseInt(b,10):nb.test(a.nodeName)||ob.test(a.nodeName)&&a.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),o.optSelected||(r.propHooks.selected={get:function(a){var b=a.parentNode;return b&&b.parentNode&&b.parentNode.selectedIndex,null},set:function(a){var b=a.parentNode;b&&(b.selectedIndex,b.parentNode&&b.parentNode.selectedIndex)}}),r.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){r.propFix[this.toLowerCase()]=this});function pb(a){var b=a.match(L)||[];return b.join(" ")}function qb(a){return a.getAttribute&&a.getAttribute("class")||""}r.fn.extend({addClass:function(a){var b,c,d,e,f,g,h,i=0;if(r.isFunction(a))return this.each(function(b){r(this).addClass(a.call(this,b,qb(this)))});if("string"==typeof a&&a){b=a.match(L)||[];while(c=this[i++])if(e=qb(c),d=1===c.nodeType&
&" "+pb(e)+" "){g=0;while(f=b[g++])d.indexOf(" "+f+" ")<0&&(d+=f+" ");h=pb(d),e!==h&&c.setAttribute("class",h)}}return this},removeClass:function(a){var b,c,d,e,f,g,h,i=0;if(r.isFunction(a))return this.each(function(b){r(this).removeClass(a.call(this,b,qb(this)))});if(!arguments.length)return this.attr("class","");if("string"==typeof a&&a){b=a.match(L)||[];while(c=this[i++])if(e=qb(c),d=1===c.nodeType&&" "+pb(e)+" "){g=0;while(f=b[g++])while(d.indexOf(" "+f+" ")>-1)d=d.replace(" "+f+" "," ");h=pb(d),e!==h&&c.setAttribute("class",h)}}return this},toggleClass:function(a,b){var c=typeof a;return"boolean"==typeof b&&"string"===c?b?this.addClass(a):this.removeClass(a):r.isFunction(a)?this.each(function(c){r(this).toggleClass(a.call(this,c,qb(this),b),b)}):this.each(function(){var b,d,e,f;if("string"===c){d=0,e=r(this),f=a.match(L)||[];while(b=f[d++])e.hasClass(b)?e.removeClass(b):e.addClass(b)}else void 0!==a&&"boolean"!==c||(b=qb(this),b&&W.set(this,"__className__",b),this.setAttribute&
&this.setAttribute("class",b||a===!1?"":W.get(this,"__className__")||""))})},hasClass:function(a){var b,c,d=0;b=" "+a+" ";while(c=this[d++])if(1===c.nodeType&&(" "+pb(qb(c))+" ").indexOf(b)>-1)return!0;return!1}});var rb=/\r/g;r.fn.extend({val:function(a){var b,c,d,e=this[0];{if(arguments.length)return d=r.isFunction(a),this.each(function(c){var e;1===this.nodeType&&(e=d?a.call(this,c,r(this).val()):a,null==e?e="":"number"==typeof e?e+="":Array.isArray(e)&&(e=r.map(e,function(a){return null==a?"":a+""})),b=r.valHooks[this.type]||r.valHooks[this.nodeName.toLowerCase()],b&&"set"in b&&void 0!==b.set(this,e,"value")||(this.value=e))});if(e)return b=r.valHooks[e.type]||r.valHooks[e.nodeName.toLowerCase()],b&&"get"in b&&void 0!==(c=b.get(e,"value"))?c:(c=e.value,"string"==typeof c?c.replace(rb,""):null==c?"":c)}}}),r.extend({valHooks:{option:{get:function(a){var b=r.find.attr(a,"value");return null!=b?b:pb(r.text(a))}},select:{get:function(a){var b,c,d,e=a.options,f=a.selectedIndex,g="sel
ect-one"===a.type,h=g?null:[],i=g?f+1:e.length;for(d=f<0?i:g?f:0;d<i;d++)if(c=e[d],(c.selected||d===f)&&!c.disabled&&(!c.parentNode.disabled||!B(c.parentNode,"optgroup"))){if(b=r(c).val(),g)return b;h.push(b)}return h},set:function(a,b){var c,d,e=a.options,f=r.makeArray(b),g=e.length;while(g--)d=e[g],(d.selected=r.inArray(r.valHooks.option.get(d),f)>-1)&&(c=!0);return c||(a.selectedIndex=-1),f}}}}),r.each(["radio","checkbox"],function(){r.valHooks[this]={set:function(a,b){if(Array.isArray(b))return a.checked=r.inArray(r(a).val(),b)>-1}},o.checkOn||(r.valHooks[this].get=function(a){return null===a.getAttribute("value")?"on":a.value})});var sb=/^(?:focusinfocus|focusoutblur)$/;r.extend(r.event,{trigger:function(b,c,e,f){var g,h,i,j,k,m,n,o=[e||d],p=l.call(b,"type")?b.type:b,q=l.call(b,"namespace")?b.namespace.split("."):[];if(h=i=e=e||d,3!==e.nodeType&&8!==e.nodeType&&!sb.test(p+r.event.triggered)&&(p.indexOf(".")>-1&&(q=p.split("."),p=q.shift(),q.sort()),k=p.indexOf(":")<0&&"on"+p,b=
b[r.expando]?b:new r.Event(p,"object"==typeof b&&b),b.isTrigger=f?2:3,b.namespace=q.join("."),b.rnamespace=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=e),c=null==c?[b]:r.makeArray(c,[b]),n=r.event.special[p]||{},f||!n.trigger||n.trigger.apply(e,c)!==!1)){if(!f&&!n.noBubble&&!r.isWindow(e)){for(j=n.delegateType||p,sb.test(j+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),i=h;i===(e.ownerDocument||d)&&o.push(i.defaultView||i.parentWindow||a)}g=0;while((h=o[g++])&&!b.isPropagationStopped())b.type=g>1?j:n.bindType||p,m=(W.get(h,"events")||{})[b.type]&&W.get(h,"handle"),m&&m.apply(h,c),m=k&&h[k],m&&m.apply&&U(h)&&(b.result=m.apply(h,c),b.result===!1&&b.preventDefault());return b.type=p,f||b.isDefaultPrevented()||n._default&&n._default.apply(o.pop(),c)!==!1||!U(e)||k&&r.isFunction(e[p])&&!r.isWindow(e)&&(i=e[k],i&&(e[k]=null),r.event.triggered=p,e[p](),r.event.triggered=void 0,i&&(e[k]=i)),b.result}},simulate:function(a,b,c)
{var d=r.extend(new r.Event,c,{type:a,isSimulated:!0});r.event.trigger(d,null,b)}}),r.fn.extend({trigger:function(a,b){return this.each(function(){r.event.trigger(a,b,this)})},triggerHandler:function(a,b){var c=this[0];if(c)return r.event.trigger(a,b,c,!0)}}),r.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(a,b){r.fn[b]=function(a,c){return arguments.length>0?this.on(b,null,a,c):this.trigger(b)}}),r.fn.extend({hover:function(a,b){return this.mouseenter(a).mouseleave(b||a)}}),o.focusin="onfocusin"in a,o.focusin||r.each({focus:"focusin",blur:"focusout"},function(a,b){var c=function(a){r.event.simulate(b,a.target,r.event.fix(a))};r.event.special[b]={setup:function(){var d=this.ownerDocument||this,e=W.access(d,b);e||d.addEventListener(a,c,!0),W.access(d,b,(e||0)+1)},teardown:function(){var d=this.ownerDocument||this,e=W.access(
d,b)-1;e?W.access(d,b,e):(d.removeEventListener(a,c,!0),W.remove(d,b))}}});var tb=a.location,ub=r.now(),vb=/\?/;r.parseXML=function(b){var c;if(!b||"string"!=typeof b)return null;try{c=(new a.DOMParser).parseFromString(b,"text/xml")}catch(d){c=void 0}return c&&!c.getElementsByTagName("parsererror").length||r.error("Invalid XML: "+b),c};var wb=/\[\]$/,xb=/\r?\n/g,yb=/^(?:submit|button|image|reset|file)$/i,zb=/^(?:input|select|textarea|keygen)/i;function Ab(a,b,c,d){var e;if(Array.isArray(b))r.each(b,function(b,e){c||wb.test(a)?d(a,e):Ab(a+"["+("object"==typeof e&&null!=e?b:"")+"]",e,c,d)});else if(c||"object"!==r.type(b))d(a,b);else for(e in b)Ab(a+"["+e+"]",b[e],c,d)}r.param=function(a,b){var c,d=[],e=function(a,b){var c=r.isFunction(b)?b():b;d[d.length]=encodeURIComponent(a)+"="+encodeURIComponent(null==c?"":c)};if(Array.isArray(a)||a.jquery&&!r.isPlainObject(a))r.each(a,function(){e(this.name,this.value)});else for(c in a)Ab(c,a[c],b,e);return d.join("&")},r.fn.extend({serialize:f
unction(){return r.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var a=r.prop(this,"elements");return a?r.makeArray(a):this}).filter(function(){var a=this.type;return this.name&&!r(this).is(":disabled")&&zb.test(this.nodeName)&&!yb.test(a)&&(this.checked||!ja.test(a))}).map(function(a,b){var c=r(this).val();return null==c?null:Array.isArray(c)?r.map(c,function(a){return{name:b.name,value:a.replace(xb,"\r\n")}}):{name:b.name,value:c.replace(xb,"\r\n")}}).get()}});var Bb=/%20/g,Cb=/#.*$/,Db=/([?&])_=[^&]*/,Eb=/^(.*?):[ \t]*([^\r\n]*)$/gm,Fb=/^(?:about|app|app-storage|.+-extension|file|res|widget):$/,Gb=/^(?:GET|HEAD)$/,Hb=/^\/\//,Ib={},Jb={},Kb="*/".concat("*"),Lb=d.createElement("a");Lb.href=tb.href;function Mb(a){return function(b,c){"string"!=typeof b&&(c=b,b="*");var d,e=0,f=b.toLowerCase().match(L)||[];if(r.isFunction(c))while(d=f[e++])"+"===d[0]?(d=d.slice(1)||"*",(a[d]=a[d]||[]).unshift(c)):(a[d]=a[d]||[]).push(c)}}function Nb(a,b,c,d){var e
={},f=a===Jb;function g(h){var i;return e[h]=!0,r.each(a[h]||[],function(a,h){var j=h(b,c,d);return"string"!=typeof j||f||e[j]?f?!(i=j):void 0:(b.dataTypes.unshift(j),g(j),!1)}),i}return g(b.dataTypes[0])||!e["*"]&&g("*")}function Ob(a,b){var c,d,e=r.ajaxSettings.flatOptions||{};for(c in b)void 0!==b[c]&&((e[c]?a:d||(d={}))[c]=b[c]);return d&&r.extend(!0,a,d),a}function Pb(a,b,c){var d,e,f,g,h=a.contents,i=a.dataTypes;while("*"===i[0])i.shift(),void 0===d&&(d=a.mimeType||b.getResponseHeader("Content-Type"));if(d)for(e in h)if(h[e]&&h[e].test(d)){i.unshift(e);break}if(i[0]in c)f=i[0];else{for(e in c){if(!i[0]||a.converters[e+" "+i[0]]){f=e;break}g||(g=e)}f=f||g}if(f)return f!==i[0]&&i.unshift(f),c[f]}function Qb(a,b,c,d){var e,f,g,h,i,j={},k=a.dataTypes.slice();if(k[1])for(g in a.converters)j[g.toLowerCase()]=a.converters[g];f=k.shift();while(f)if(a.responseFields[f]&&(c[a.responseFields[f]]=b),!i&&d&&a.dataFilter&&(b=a.dataFilter(b,a.dataType)),i=f,f=k.shift())if("*"===f)f=i;else if
("*"!==i&&i!==f){if(g=j[i+" "+f]||j["* "+f],!g)for(e in j)if(h=e.split(" "),h[1]===f&&(g=j[i+" "+h[0]]||j["* "+h[0]])){g===!0?g=j[e]:j[e]!==!0&&(f=h[0],k.unshift(h[1]));break}if(g!==!0)if(g&&a["throws"])b=g(b);else try{b=g(b)}catch(l){return{state:"parsererror",error:g?l:"No conversion from "+i+" to "+f}}}return{state:"success",data:b}}r.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:tb.href,type:"GET",isLocal:Fb.test(tb.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":Kb,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":r.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(a,b){return b?Ob(Ob(a,r.ajaxSettings),b):Ob(r.ajaxSetting
s,a)},ajaxPrefilter:Mb(Ib),ajaxTransport:Mb(Jb),ajax:function(b,c){"object"==typeof b&&(c=b,b=void 0),c=c||{};var e,f,g,h,i,j,k,l,m,n,o=r.ajaxSetup({},c),p=o.context||o,q=o.context&&(p.nodeType||p.jquery)?r(p):r.event,s=r.Deferred(),t=r.Callbacks("once memory"),u=o.statusCode||{},v={},w={},x="canceled",y={readyState:0,getResponseHeader:function(a){var b;if(k){if(!h){h={};while(b=Eb.exec(g))h[b[1].toLowerCase()]=b[2]}b=h[a.toLowerCase()]}return null==b?null:b},getAllResponseHeaders:function(){return k?g:null},setRequestHeader:function(a,b){return null==k&&(a=w[a.toLowerCase()]=w[a.toLowerCase()]||a,v[a]=b),this},overrideMimeType:function(a){return null==k&&(o.mimeType=a),this},statusCode:function(a){var b;if(a)if(k)y.always(a[y.status]);else for(b in a)u[b]=[u[b],a[b]];return this},abort:function(a){var b=a||x;return e&&e.abort(b),A(0,b),this}};if(s.promise(y),o.url=((b||o.url||tb.href)+"").replace(Hb,tb.protocol+"//"),o.type=c.method||c.type||o.method||o.type,o.dataTypes=(o.dataType
||"*").toLowerCase().match(L)||[""],null==o.crossDomain){j=d.createElement("a");try{j.href=o.url,j.href=j.href,o.crossDomain=Lb.protocol+"//"+Lb.host!=j.protocol+"//"+j.host}catch(z){o.crossDomain=!0}}if(o.data&&o.processData&&"string"!=typeof o.data&&(o.data=r.param(o.data,o.traditional)),Nb(Ib,o,c,y),k)return y;l=r.event&&o.global,l&&0===r.active++&&r.event.trigger("ajaxStart"),o.type=o.type.toUpperCase(),o.hasContent=!Gb.test(o.type),f=o.url.replace(Cb,""),o.hasContent?o.data&&o.processData&&0===(o.contentType||"").indexOf("application/x-www-form-urlencoded")&&(o.data=o.data.replace(Bb,"+")):(n=o.url.slice(f.length),o.data&&(f+=(vb.test(f)?"&":"?")+o.data,delete o.data),o.cache===!1&&(f=f.replace(Db,"$1"),n=(vb.test(f)?"&":"?")+"_="+ub++ +n),o.url=f+n),o.ifModified&&(r.lastModified[f]&&y.setRequestHeader("If-Modified-Since",r.lastModified[f]),r.etag[f]&&y.setRequestHeader("If-None-Match",r.etag[f])),(o.data&&o.hasContent&&o.contentType!==!1||c.contentType)&&y.setRequestHeader("Co
ntent-Type",o.contentType),y.setRequestHeader("Accept",o.dataTypes[0]&&o.accepts[o.dataTypes[0]]?o.accepts[o.dataTypes[0]]+("*"!==o.dataTypes[0]?", "+Kb+"; q=0.01":""):o.accepts["*"]);for(m in o.headers)y.setRequestHeader(m,o.headers[m]);if(o.beforeSend&&(o.beforeSend.call(p,y,o)===!1||k))return y.abort();if(x="abort",t.add(o.complete),y.done(o.success),y.fail(o.error),e=Nb(Jb,o,c,y)){if(y.readyState=1,l&&q.trigger("ajaxSend",[y,o]),k)return y;o.async&&o.timeout>0&&(i=a.setTimeout(function(){y.abort("timeout")},o.timeout));try{k=!1,e.send(v,A)}catch(z){if(k)throw z;A(-1,z)}}else A(-1,"No Transport");function A(b,c,d,h){var j,m,n,v,w,x=c;k||(k=!0,i&&a.clearTimeout(i),e=void 0,g=h||"",y.readyState=b>0?4:0,j=b>=200&&b<300||304===b,d&&(v=Pb(o,y,d)),v=Qb(o,v,y,j),j?(o.ifModified&&(w=y.getResponseHeader("Last-Modified"),w&&(r.lastModified[f]=w),w=y.getResponseHeader("etag"),w&&(r.etag[f]=w)),204===b||"HEAD"===o.type?x="nocontent":304===b?x="notmodified":(x=v.state,m=v.data,n=v.error,j=!n)
):(n=x,!b&&x||(x="error",b<0&&(b=0))),y.status=b,y.statusText=(c||x)+"",j?s.resolveWith(p,[m,x,y]):s.rejectWith(p,[y,x,n]),y.statusCode(u),u=void 0,l&&q.trigger(j?"ajaxSuccess":"ajaxError",[y,o,j?m:n]),t.fireWith(p,[y,x]),l&&(q.trigger("ajaxComplete",[y,o]),--r.active||r.event.trigger("ajaxStop")))}return y},getJSON:function(a,b,c){return r.get(a,b,c,"json")},getScript:function(a,b){return r.get(a,void 0,b,"script")}}),r.each(["get","post"],function(a,b){r[b]=function(a,c,d,e){return r.isFunction(c)&&(e=e||d,d=c,c=void 0),r.ajax(r.extend({url:a,type:b,dataType:e,data:c,success:d},r.isPlainObject(a)&&a))}}),r._evalUrl=function(a){return r.ajax({url:a,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,"throws":!0})},r.fn.extend({wrapAll:function(a){var b;return this[0]&&(r.isFunction(a)&&(a=a.call(this[0])),b=r(a,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&b.insertBefore(this[0]),b.map(function(){var a=this;while(a.firstElementChild)a=a.firstElementChild;return a}
).append(this)),this},wrapInner:function(a){return r.isFunction(a)?this.each(function(b){r(this).wrapInner(a.call(this,b))}):this.each(function(){var b=r(this),c=b.contents();c.length?c.wrapAll(a):b.append(a)})},wrap:function(a){var b=r.isFunction(a);return this.each(function(c){r(this).wrapAll(b?a.call(this,c):a)})},unwrap:function(a){return this.parent(a).not("body").each(function(){r(this).replaceWith(this.childNodes)}),this}}),r.expr.pseudos.hidden=function(a){return!r.expr.pseudos.visible(a)},r.expr.pseudos.visible=function(a){return!!(a.offsetWidth||a.offsetHeight||a.getClientRects().length)},r.ajaxSettings.xhr=function(){try{return new a.XMLHttpRequest}catch(b){}};var Rb={0:200,1223:204},Sb=r.ajaxSettings.xhr();o.cors=!!Sb&&"withCredentials"in Sb,o.ajax=Sb=!!Sb,r.ajaxTransport(function(b){var c,d;if(o.cors||Sb&&!b.crossDomain)return{send:function(e,f){var g,h=b.xhr();if(h.open(b.type,b.url,b.async,b.username,b.password),b.xhrFields)for(g in b.xhrFields)h[g]=b.xhrFields[g];b.m
imeType&&h.overrideMimeType&&h.overrideMimeType(b.mimeType),b.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest");for(g in e)h.setRequestHeader(g,e[g]);c=function(a){return function(){c&&(c=d=h.onload=h.onerror=h.onabort=h.onreadystatechange=null,"abort"===a?h.abort():"error"===a?"number"!=typeof h.status?f(0,"error"):f(h.status,h.statusText):f(Rb[h.status]||h.status,h.statusText,"text"!==(h.responseType||"text")||"string"!=typeof h.responseText?{binary:h.response}:{text:h.responseText},h.getAllResponseHeaders()))}},h.onload=c(),d=h.onerror=c("error"),void 0!==h.onabort?h.onabort=d:h.onreadystatechange=function(){4===h.readyState&&a.setTimeout(function(){c&&d()})},c=c("abort");try{h.send(b.hasContent&&b.data||null)}catch(i){if(c)throw i}},abort:function(){c&&c()}}}),r.ajaxPrefilter(function(a){a.crossDomain&&(a.contents.script=!1)}),r.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},conte
nts:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(a){return r.globalEval(a),a}}}),r.ajaxPrefilter("script",function(a){void 0===a.cache&&(a.cache=!1),a.crossDomain&&(a.type="GET")}),r.ajaxTransport("script",function(a){if(a.crossDomain){var b,c;return{send:function(e,f){b=r("<script>").prop({charset:a.scriptCharset,src:a.url}).on("load error",c=function(a){b.remove(),c=null,a&&f("error"===a.type?404:200,a.type)}),d.head.appendChild(b[0])},abort:function(){c&&c()}}}});var Tb=[],Ub=/(=)\?(?=&|$)|\?\?/;r.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var a=Tb.pop()||r.expando+"_"+ub++;return this[a]=!0,a}}),r.ajaxPrefilter("json jsonp",function(b,c,d){var e,f,g,h=b.jsonp!==!1&&(Ub.test(b.url)?"url":"string"==typeof b.data&&0===(b.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ub.test(b.data)&&"data");if(h||"jsonp"===b.dataTypes[0])return e=b.jsonpCallback=r.isFunction(b.jsonpCallback)?b.jsonpCallback():b.jsonpCallback,h?b[h]=b[h].replace(Ub
,"$1"+e):b.jsonp!==!1&&(b.url+=(vb.test(b.url)?"&":"?")+b.jsonp+"="+e),b.converters["script json"]=function(){return g||r.error(e+" was not called"),g[0]},b.dataTypes[0]="json",f=a[e],a[e]=function(){g=arguments},d.always(function(){void 0===f?r(a).removeProp(e):a[e]=f,b[e]&&(b.jsonpCallback=c.jsonpCallback,Tb.push(e)),g&&r.isFunction(f)&&f(g[0]),g=f=void 0}),"script"}),o.createHTMLDocument=function(){var a=d.implementation.createHTMLDocument("").body;return a.innerHTML="<form></form><form></form>",2===a.childNodes.length}(),r.parseHTML=function(a,b,c){if("string"!=typeof a)return[];"boolean"==typeof b&&(c=b,b=!1);var e,f,g;return b||(o.createHTMLDocument?(b=d.implementation.createHTMLDocument(""),e=b.createElement("base"),e.href=d.location.href,b.head.appendChild(e)):b=d),f=C.exec(a),g=!c&&[],f?[b.createElement(f[1])]:(f=qa([a],b,g),g&&g.length&&r(g).remove(),r.merge([],f.childNodes))},r.fn.load=function(a,b,c){var d,e,f,g=this,h=a.indexOf(" ");return h>-1&&(d=pb(a.slice(h)),a=a.sl
ice(0,h)),r.isFunction(b)?(c=b,b=void 0):b&&"object"==typeof b&&(e="POST"),g.length>0&&r.ajax({url:a,type:e||"GET",dataType:"html",data:b}).done(function(a){f=arguments,g.html(d?r("<div>").append(r.parseHTML(a)).find(d):a)}).always(c&&function(a,b){g.each(function(){c.apply(this,f||[a.responseText,b,a])})}),this},r.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(a,b){r.fn[b]=function(a){return this.on(b,a)}}),r.expr.pseudos.animated=function(a){return r.grep(r.timers,function(b){return a===b.elem}).length},r.offset={setOffset:function(a,b,c){var d,e,f,g,h,i,j,k=r.css(a,"position"),l=r(a),m={};"static"===k&&(a.style.position="relative"),h=l.offset(),f=r.css(a,"top"),i=r.css(a,"left"),j=("absolute"===k||"fixed"===k)&&(f+i).indexOf("auto")>-1,j?(d=l.position(),g=d.top,e=d.left):(g=parseFloat(f)||0,e=parseFloat(i)||0),r.isFunction(b)&&(b=b.call(a,c,r.extend({},h))),null!=b.top&&(m.top=b.top-h.top+g),null!=b.left&&(m.left=b.left-h.left+e),"using
"in b?b.using.call(a,m):l.css(m)}},r.fn.extend({offset:function(a){if(arguments.length)return void 0===a?this:this.each(function(b){r.offset.setOffset(this,a,b)});var b,c,d,e,f=this[0];if(f)return f.getClientRects().length?(d=f.getBoundingClientRect(),b=f.ownerDocument,c=b.documentElement,e=b.defaultView,{top:d.top+e.pageYOffset-c.clientTop,left:d.left+e.pageXOffset-c.clientLeft}):{top:0,left:0}},position:function(){if(this[0]){var a,b,c=this[0],d={top:0,left:0};return"fixed"===r.css(c,"position")?b=c.getBoundingClientRect():(a=this.offsetParent(),b=this.offset(),B(a[0],"html")||(d=a.offset()),d={top:d.top+r.css(a[0],"borderTopWidth",!0),left:d.left+r.css(a[0],"borderLeftWidth",!0)}),{top:b.top-d.top-r.css(c,"marginTop",!0),left:b.left-d.left-r.css(c,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var a=this.offsetParent;while(a&&"static"===r.css(a,"position"))a=a.offsetParent;return a||ra})}}),r.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function
(a,b){var c="pageYOffset"===b;r.fn[a]=function(d){return T(this,function(a,d,e){var f;return r.isWindow(a)?f=a:9===a.nodeType&&(f=a.defaultView),void 0===e?f?f[b]:a[d]:void(f?f.scrollTo(c?f.pageXOffset:e,c?e:f.pageYOffset):a[d]=e)},a,d,arguments.length)}}),r.each(["top","left"],function(a,b){r.cssHooks[b]=Pa(o.pixelPosition,function(a,c){if(c)return c=Oa(a,b),Ma.test(c)?r(a).position()[b]+"px":c})}),r.each({Height:"height",Width:"width"},function(a,b){r.each({padding:"inner"+a,content:b,"":"outer"+a},function(c,d){r.fn[d]=function(e,f){var g=arguments.length&&(c||"boolean"!=typeof e),h=c||(e===!0||f===!0?"margin":"border");return T(this,function(b,c,e){var f;return r.isWindow(b)?0===d.indexOf("outer")?b["inner"+a]:b.document.documentElement["client"+a]:9===b.nodeType?(f=b.documentElement,Math.max(b.body["scroll"+a],f["scroll"+a],b.body["offset"+a],f["offset"+a],f["client"+a])):void 0===e?r.css(b,c,h):r.style(b,c,e,h)},b,g?e:void 0,g)}})}),r.fn.extend({bind:function(a,b,c){return thi
s.on(a,null,b,c)},unbind:function(a,b){return this.off(a,null,b)},delegate:function(a,b,c,d){return this.on(b,a,c,d)},undelegate:function(a,b,c){return 1===arguments.length?this.off(a,"**"):this.off(b,a||"**",c)}}),r.holdReady=function(a){a?r.readyWait++:r.ready(!0)},r.isArray=Array.isArray,r.parseJSON=JSON.parse,r.nodeName=B,"function"==typeof define&&define.amd&&define("jquery",[],function(){return r});var Vb=a.jQuery,Wb=a.$;return r.noConflict=function(b){return a.$===r&&(a.$=Wb),b&&a.jQuery===r&&(a.jQuery=Vb),r},b||(a.jQuery=a.$=r),r});